From 62957880e4c3bf79d3a88f604e044ab5cafb4da3 Mon Sep 17 00:00:00 2001 From: koseyile Date: Mon, 1 Jul 2019 10:16:15 +0000 Subject: [PATCH 01/27] redesign MediaStream and mediaStreamTrack to support multiple track. --- Assets/Scripts/RenderStreaming.cs | 15 +++- .../Runtime/Srcipts/MediaStream.cs | 86 +++++++++++++++---- .../Runtime/Srcipts/MediaStreamTrack.cs | 34 ++++++++ .../Runtime/Srcipts/RTCPeerConnection.cs | 6 ++ .../Runtime/Srcipts/WebRTC.cs | 4 +- 5 files changed, 124 insertions(+), 21 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index 82346ea2b..2723c2e90 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -41,7 +41,7 @@ public class RenderStreaming : MonoBehaviour private Dictionary> mapChannels = new Dictionary>(); private RTCConfiguration conf; private string sessionId; - private MediaStream videoStream; + private MediaStream2 videoStream; public void Awake() { @@ -60,7 +60,16 @@ public IEnumerator Start() { yield break; } - videoStream = cam.CaptureStream(1280, 720); + cam.CreateRenderStreamTexture(1280, 720); + videoStream = new MediaStream2(); + int texCount = cam.getStreamTextureCount(); + for (int i = 0; i < texCount; ++i) + { + videoStream.AddTrack(new VideoStreamTrack(cam.getStreamTexture(i))); + } + + + signaling = new Signaling(urlSignaling); var opCreate = signaling.Create(); yield return opCreate; @@ -142,7 +151,7 @@ IEnumerator GetOffer() string pattern = @"(a=fmtp:\d+ .*level-asymmetry-allowed=.*)\r\n"; _desc.sdp = Regex.Replace(_desc.sdp, pattern, "$1;x-google-start-bitrate=16000;x-google-max-bitrate=160000\r\n"); pc.SetRemoteDescription(ref _desc); - foreach (var track in videoStream.GetTracks()) + foreach (var track in videoStream.getTracks()) { pc.AddTrack(track); } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index f96d4feac..a70b05f3b 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -6,6 +6,34 @@ namespace Unity.WebRTC { + public class MediaStream2 + { + protected List mediaStreamTrackList = new List(); + + public MediaStream2() : base() + { + + } + + public MediaStream2(MediaStreamTrack2[] tracks) : base() + { + foreach (var t in tracks) + { + mediaStreamTrackList.Add(t); + } + } + + public void AddTrack(MediaStreamTrack2 track) + { + mediaStreamTrackList.Add(track); + } + + public MediaStreamTrack2[] getTracks() + { + return mediaStreamTrackList.ToArray(); + } + } + public class MediaStream { private IntPtr self; @@ -152,36 +180,62 @@ public static void AddCleanerCallback(this GameObject obj, Action callback) } public static class CameraExtension { + internal static RenderTexture camRenderTexture; + internal static List webRTCTextures = new List(); internal static List camCopyRts = new List(); internal static bool started = false; - public static MediaStream CaptureStream(this Camera cam, int width, int height) + + public static int getStreamTextureCount(this Camera cam) + { + return webRTCTextures.Count; + } + + public static RenderTexture getStreamTexture(this Camera cam, int index) { + return webRTCTextures[index]; + } + + public static void CreateRenderStreamTexture(this Camera cam, int width, int height) { if (camCopyRts.Count > 0) { throw new NotImplementedException("Currently not allowed multiple MediaStream"); } - RenderTexture[] rts = new RenderTexture[2]; - //rts[0] for render target, rts[1] for flip and WebRTC source - rts[0] = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32); - rts[1] = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32); - rts[0].Create(); - rts[1].Create(); - camCopyRts.Add(rts); - cam.targetTexture = rts[0]; + camRenderTexture = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32); + camRenderTexture.Create(); + + int mipCount = 1; + for (int i = 1, mipLevel = 1; i <= mipCount; ++i, mipLevel *= 2) + { + RenderTexture webRtcTex = new RenderTexture(width / mipLevel, height / mipLevel, 0, RenderTextureFormat.BGRA32); + webRtcTex.Create(); + webRTCTextures.Add(webRtcTex); + } + + cam.targetTexture = camRenderTexture; cam.gameObject.AddCleanerCallback(() => { - if (rts != null) + camRenderTexture.Release(); + UnityEngine.Object.Destroy(camRenderTexture); + + foreach (var v in webRTCTextures) { - CameraExtension.RemoveRt(rts); - rts[0].Release(); - rts[1].Release(); - UnityEngine.Object.Destroy(rts[0]); - UnityEngine.Object.Destroy(rts[1]); + v.Release(); + UnityEngine.Object.Destroy(v); } + webRTCTextures.Clear(); }); started = true; - return new MediaStream(rts, WebRTC.Context.CaptureVideoStream(rts[1].GetNativeTexturePtr(), width, height)); + } + + public static MediaStream CaptureStream(this Camera cam, int width, int height) + { + cam.CreateRenderStreamTexture(width, height); + + int textureIndex = 0; + int rtcMipLevel = (int)Math.Pow(2, textureIndex); //1 2 4 8 + return new MediaStream(webRTCTextures.ToArray(), WebRTC.Context.CaptureVideoStream(webRTCTextures[textureIndex].GetNativeTexturePtr(), width/rtcMipLevel, height/rtcMipLevel)); + //return new MediaStream2(); } public static void RemoveRt(RenderTexture[] rts) { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index 30df48c7d..3e1d5de8e 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -4,6 +4,40 @@ namespace Unity.WebRTC { + public class MediaStreamTrack2 + { + public IntPtr nativePtr; + public string id; + public TrackKind kind; + public MediaStreamTrack2() + { + + } + } + + public class VideoStreamTrack : MediaStreamTrack2 + { + public VideoStreamTrack(RenderTexture rt) : base() + { + IntPtr nativeVideoStreamPtr = WebRTC.Context.CaptureVideoStream(rt.GetNativeTexturePtr(), rt.width, rt.height); + string nativeVideoStreamID = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(nativeVideoStreamPtr)); + + int trackSize = 0; + IntPtr tracksNativePtr = NativeMethods.MediaStreamGetVideoTracks(nativeVideoStreamPtr, ref trackSize); + IntPtr[] tracksPtr = new IntPtr[trackSize]; + Marshal.Copy(tracksNativePtr, tracksPtr, 0, trackSize); + Marshal.FreeCoTaskMem(tracksNativePtr); + + nativePtr = tracksPtr[0]; + kind = NativeMethods.MediaStreamTrackGetKind(nativePtr); + id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(nativePtr)); + + } + } + + + + public class MediaStreamTrack { internal IntPtr self; diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs index 9c9e74469..b4d3f3efb 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs @@ -192,6 +192,12 @@ public RTCRtpSender AddTrack(MediaStreamTrack track) { return new RTCRtpSender(NativeMethods.PeerConnectionAddTrack(self, track.self)); } + + public RTCRtpSender AddTrack(MediaStreamTrack2 track) + { + return new RTCRtpSender(NativeMethods.PeerConnectionAddTrack(self, track.nativePtr)); + } + public void RemoveTrack(RTCRtpSender sender) { NativeMethods.PeerConnectionRemoveTrack(self, sender.self); diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs index 1c919b57a..0b1caadf3 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs @@ -222,9 +222,9 @@ public static IEnumerator Update() if (CameraExtension.started) { //Blit is for DirectX Rendering API Only - foreach(var rts in CameraExtension.camCopyRts) + foreach(var rt in CameraExtension.webRTCTextures) { - Graphics.Blit(rts[0], rts[1], flipMat); + Graphics.Blit(CameraExtension.camRenderTexture, rt, flipMat); } GL.IssuePluginEvent(NativeMethods.GetRenderEventFunc(), 0); } From f7310c638b5d2903a423cb472157b30a410353b7 Mon Sep 17 00:00:00 2001 From: koseyile Date: Mon, 1 Jul 2019 12:59:17 +0000 Subject: [PATCH 02/27] delete member variable videoTrackToRts and AudioTracks from mediaStream. --- Assets/Scripts/RenderStreaming.cs | 4 +- .../Runtime/Srcipts/MediaStream.cs | 255 ++++++++++-------- .../Runtime/Srcipts/MediaStreamTrack.cs | 118 ++++---- .../Runtime/Srcipts/RTCPeerConnection.cs | 5 - .../Samples/Example/AddMediaStream.cs | 8 +- .../Tests/Runtime/MediaStreamTest.cs | 14 +- 6 files changed, 226 insertions(+), 178 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index 2723c2e90..19e97fd7a 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -41,7 +41,7 @@ public class RenderStreaming : MonoBehaviour private Dictionary> mapChannels = new Dictionary>(); private RTCConfiguration conf; private string sessionId; - private MediaStream2 videoStream; + private MediaStream videoStream; public void Awake() { @@ -61,7 +61,7 @@ public IEnumerator Start() yield break; } cam.CreateRenderStreamTexture(1280, 720); - videoStream = new MediaStream2(); + videoStream = new MediaStream(); int texCount = cam.getStreamTextureCount(); for (int i = 0; i < texCount; ++i) { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index a70b05f3b..c343c64d7 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -6,16 +6,38 @@ namespace Unity.WebRTC { - public class MediaStream2 + public class MediaStream { - protected List mediaStreamTrackList = new List(); + public IntPtr self; + public string id; + protected List mediaStreamTrackList = new List(); - public MediaStream2() : base() + public MediaStream() : base() { } - public MediaStream2(MediaStreamTrack2[] tracks) : base() + internal MediaStream(IntPtr ptr) + { + self = ptr; + id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); + int trackSize = 0; + IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(self, ref trackSize); + IntPtr[] tracksPtr = new IntPtr[trackSize]; + Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); + //TODO: Linux compatibility + Marshal.FreeCoTaskMem(trackNativePtr); + + for (int i = 0; i < trackSize; i++) + { + MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); + //track.stopTrack += StopTrack; + //track.getRts += GetRts; + mediaStreamTrackList.Add(track); + } + } + + public MediaStream(MediaStreamTrack[] tracks) : base() { foreach (var t in tracks) { @@ -23,133 +45,133 @@ public MediaStream2(MediaStreamTrack2[] tracks) : base() } } - public void AddTrack(MediaStreamTrack2 track) + public void AddTrack(MediaStreamTrack track) { mediaStreamTrackList.Add(track); } - public MediaStreamTrack2[] getTracks() + public MediaStreamTrack[] getTracks() { return mediaStreamTrackList.ToArray(); } } - public class MediaStream - { - private IntPtr self; - private string id; - public string Id { get => id; private set { } } + //public class MediaStream + //{ + // private IntPtr self; + // private string id; + // public string Id { get => id; private set { } } - private Dictionary VideoTrackToRts; - private List AudioTracks; + // private Dictionary VideoTrackToRts; + // private List AudioTracks; - private void StopTrack(MediaStreamTrack track) - { + // //private void StopTrack(MediaStreamTrack track) + // //{ - if (track.Kind == TrackKind.Video) - { - NativeMethods.StopMediaStreamTrack(track.self); - RenderTexture[] rts = VideoTrackToRts[track]; - if (rts != null) - { - CameraExtension.RemoveRt(rts); - rts[0].Release(); - rts[1].Release(); - UnityEngine.Object.Destroy(rts[0]); - UnityEngine.Object.Destroy(rts[1]); - } - } - else - { - Audio.Stop(); - } + // // if (track.kind == TrackKind.Video) + // // { + // // NativeMethods.StopMediaStreamTrack(track.nativePtr); + // // RenderTexture[] rts = VideoTrackToRts[track]; + // // if (rts != null) + // // { + // // CameraExtension.RemoveRt(rts); + // // rts[0].Release(); + // // rts[1].Release(); + // // UnityEngine.Object.Destroy(rts[0]); + // // UnityEngine.Object.Destroy(rts[1]); + // // } + // // } + // // else + // // { + // // Audio.Stop(); + // // } - } - private RenderTexture[] GetRts(MediaStreamTrack track) - { - return VideoTrackToRts[track]; - } - public MediaStreamTrack[] GetTracks() - { - MediaStreamTrack[] tracks = new MediaStreamTrack[VideoTrackToRts.Keys.Count + AudioTracks.Count]; - AudioTracks.CopyTo(tracks, 0); - VideoTrackToRts.Keys.CopyTo(tracks, AudioTracks.Count); - return tracks; - } - public MediaStreamTrack[] GetAudioTracks() - { - return AudioTracks.ToArray(); - } - public MediaStreamTrack[] GetVideoTracks() - { - MediaStreamTrack[] tracks = new MediaStreamTrack[VideoTrackToRts.Keys.Count]; - VideoTrackToRts.Keys.CopyTo(tracks, 0); - return tracks; - } + // //} + // private RenderTexture[] GetRts(MediaStreamTrack track) + // { + // return VideoTrackToRts[track]; + // } + // public MediaStreamTrack[] GetTracks() + // { + // MediaStreamTrack[] tracks = new MediaStreamTrack[VideoTrackToRts.Keys.Count + AudioTracks.Count]; + // AudioTracks.CopyTo(tracks, 0); + // VideoTrackToRts.Keys.CopyTo(tracks, AudioTracks.Count); + // return tracks; + // } + // //public MediaStreamTrack[] GetAudioTracks() + // //{ + // // return AudioTracks.ToArray(); + // //} + // //public MediaStreamTrack[] GetVideoTracks() + // //{ + // // MediaStreamTrack[] tracks = new MediaStreamTrack[VideoTrackToRts.Keys.Count]; + // // VideoTrackToRts.Keys.CopyTo(tracks, 0); + // // return tracks; + // //} - public void AddTrack(MediaStreamTrack track) - { - if(track.Kind == TrackKind.Video) - { - VideoTrackToRts[track] = track.getRts(track); - } - else - { - AudioTracks.Add(track); - } - NativeMethods.MediaStreamAddTrack(self, track.self); - } - public void RemoveTrack(MediaStreamTrack track) - { - NativeMethods.MediaStreamRemoveTrack(self, track.self); - } - //for camera CaptureStream - internal MediaStream(RenderTexture[] rts, IntPtr ptr) - { - self = ptr; - id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); - VideoTrackToRts = new Dictionary(); - AudioTracks = new List(); - //get initial tracks - int trackSize = 0; - IntPtr tracksNativePtr = NativeMethods.MediaStreamGetVideoTracks(self, ref trackSize); - IntPtr[] tracksPtr = new IntPtr[trackSize]; - Marshal.Copy(tracksNativePtr, tracksPtr, 0, trackSize); - //TODO: Linux compatibility - Marshal.FreeCoTaskMem(tracksNativePtr); - for (int i = 0; i < trackSize; i++) - { - MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); - track.stopTrack += StopTrack; - track.getRts += GetRts; - VideoTrackToRts[track] = rts; - } - } - //for audio CaptureStream - internal MediaStream(IntPtr ptr) - { - self = ptr; - id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); - VideoTrackToRts = new Dictionary(); - AudioTracks = new List(); - //get initial tracks - int trackSize = 0; - IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(self, ref trackSize); - IntPtr[] tracksPtr = new IntPtr[trackSize]; - Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); - //TODO: Linux compatibility - Marshal.FreeCoTaskMem(trackNativePtr); + // //public void AddTrack(MediaStreamTrack track) + // //{ + // // if(track.kind == TrackKind.Video) + // // { + // // VideoTrackToRts[track] = track.getRts(track); + // // } + // // else + // // { + // // AudioTracks.Add(track); + // // } + // // NativeMethods.MediaStreamAddTrack(self, track.self); + // //} + // //public void RemoveTrack(MediaStreamTrack track) + // //{ + // // NativeMethods.MediaStreamRemoveTrack(self, track.self); + // //} + // //for camera CaptureStream + // internal MediaStream(RenderTexture[] rts, IntPtr ptr) + // { + // self = ptr; + // id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); + // VideoTrackToRts = new Dictionary(); + // AudioTracks = new List(); + // //get initial tracks + // int trackSize = 0; + // IntPtr tracksNativePtr = NativeMethods.MediaStreamGetVideoTracks(self, ref trackSize); + // IntPtr[] tracksPtr = new IntPtr[trackSize]; + // Marshal.Copy(tracksNativePtr, tracksPtr, 0, trackSize); + // //TODO: Linux compatibility + // Marshal.FreeCoTaskMem(tracksNativePtr); + // for (int i = 0; i < trackSize; i++) + // { + // MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); + // //track.stopTrack += StopTrack; + // //track.getRts += GetRts; + // VideoTrackToRts[track] = rts; + // } + // } + // //for audio CaptureStream + // internal MediaStream(IntPtr ptr) + // { + // self = ptr; + // id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); + // VideoTrackToRts = new Dictionary(); + // AudioTracks = new List(); + // //get initial tracks + // int trackSize = 0; + // IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(self, ref trackSize); + // IntPtr[] tracksPtr = new IntPtr[trackSize]; + // Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); + // //TODO: Linux compatibility + // Marshal.FreeCoTaskMem(trackNativePtr); - for (int i = 0; i < trackSize; i++) - { - MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); - track.stopTrack += StopTrack; - track.getRts += GetRts; - AudioTracks.Add(track); - } - } + // for (int i = 0; i < trackSize; i++) + // { + // MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); + // //track.stopTrack += StopTrack; + // //track.getRts += GetRts; + // AudioTracks.Add(track); + // } + // } - } + //} internal class Cleaner : MonoBehaviour { private Action onDestroy; @@ -234,8 +256,7 @@ public static MediaStream CaptureStream(this Camera cam, int width, int height) int textureIndex = 0; int rtcMipLevel = (int)Math.Pow(2, textureIndex); //1 2 4 8 - return new MediaStream(webRTCTextures.ToArray(), WebRTC.Context.CaptureVideoStream(webRTCTextures[textureIndex].GetNativeTexturePtr(), width/rtcMipLevel, height/rtcMipLevel)); - //return new MediaStream2(); + return new MediaStream(WebRTC.Context.CaptureVideoStream(webRTCTextures[textureIndex].GetNativeTexturePtr(), width/rtcMipLevel, height/rtcMipLevel)); } public static void RemoveRt(RenderTexture[] rts) { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index 3e1d5de8e..7b070f0d1 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -4,18 +4,40 @@ namespace Unity.WebRTC { - public class MediaStreamTrack2 + public class MediaStreamTrack { public IntPtr nativePtr; public string id; public TrackKind kind; - public MediaStreamTrack2() + public MediaStreamTrack() { } + + internal MediaStreamTrack(IntPtr ptr) + { + nativePtr = ptr; + kind = NativeMethods.MediaStreamTrackGetKind(nativePtr); + id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(nativePtr)); + } + + //public bool Enabled + //{ + // get{return NativeMethods.MediaStreamTrackGetEnabled(nativePtr);} + // set{NativeMethods.MediaStreamTrackSetEnabled(nativePtr, value);} + //} + //public TrackState ReadyState + //{ + // get + // {return NativeMethods.MediaStreamTrackGetReadyState(nativePtr);} + // private set { } + //} + + //public TrackKind Kind { get => kind; private set { } } + //public string Id { get => id; private set { } } } - public class VideoStreamTrack : MediaStreamTrack2 + public class VideoStreamTrack : MediaStreamTrack { public VideoStreamTrack(RenderTexture rt) : base() { @@ -38,51 +60,51 @@ public VideoStreamTrack(RenderTexture rt) : base() - public class MediaStreamTrack - { - internal IntPtr self; - private TrackKind kind; - private string id; - private bool enabled; - private TrackState readyState; - internal Action stopTrack; - internal Func getRts; - - public bool Enabled - { - get - { - return NativeMethods.MediaStreamTrackGetEnabled(self); - } - set - { - NativeMethods.MediaStreamTrackSetEnabled(self, value); - } - } - public TrackState ReadyState - { - get - { - return NativeMethods.MediaStreamTrackGetReadyState(self); - } - private set { } - } - - public TrackKind Kind { get => kind; private set { } } - public string Id { get => id; private set { } } - - internal MediaStreamTrack(IntPtr ptr) - { - self = ptr; - kind = NativeMethods.MediaStreamTrackGetKind(self); - id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(self)); - } - //Disassociate track from its source(video or audio), not for destroying the track - public void Stop() - { - stopTrack(this); - } - } + //public class MediaStreamTrack + //{ + // internal IntPtr self; + // private TrackKind kind; + // private string id; + // private bool enabled; + // private TrackState readyState; + // internal Action stopTrack; + // internal Func getRts; + + // public bool Enabled + // { + // get + // { + // return NativeMethods.MediaStreamTrackGetEnabled(self); + // } + // set + // { + // NativeMethods.MediaStreamTrackSetEnabled(self, value); + // } + // } + // public TrackState ReadyState + // { + // get + // { + // return NativeMethods.MediaStreamTrackGetReadyState(self); + // } + // private set { } + // } + + // public TrackKind Kind { get => kind; private set { } } + // public string Id { get => id; private set { } } + + // internal MediaStreamTrack(IntPtr ptr) + // { + // self = ptr; + // kind = NativeMethods.MediaStreamTrackGetKind(self); + // id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(self)); + // } + // //Disassociate track from its source(video or audio), not for destroying the track + // public void Stop() + // { + // stopTrack(this); + // } + //} public enum TrackKind { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs index b4d3f3efb..ee9b9b975 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs @@ -189,11 +189,6 @@ public void Close() } public RTCRtpSender AddTrack(MediaStreamTrack track) - { - return new RTCRtpSender(NativeMethods.PeerConnectionAddTrack(self, track.self)); - } - - public RTCRtpSender AddTrack(MediaStreamTrack2 track) { return new RTCRtpSender(NativeMethods.PeerConnectionAddTrack(self, track.nativePtr)); } diff --git a/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs b/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs index 86410f31e..5e6b609f9 100644 --- a/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs +++ b/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs @@ -159,11 +159,11 @@ void Pc2OnIceCandidate(RTCIceCandidate candidate) } public void AddTracks() { - foreach (var track in audioStream.GetTracks()) + foreach (var track in audioStream.getTracks()) { pc1Senders.Add (pc1.AddTrack(track)); } - foreach(var track in videoStream.GetTracks()) + foreach(var track in videoStream.getTracks()) { pc1Senders.Add(pc1.AddTrack(track)); } @@ -232,8 +232,8 @@ void OnTrack(RTCPeerConnection pc, RTCTrackEvent e) { pc2Senders.Add(pc.AddTrack(e.Track)); trackInfos.Append($"{GetName(pc)} receives remote track:\r\n"); - trackInfos.Append($"Track kind: {e.Track.Kind}\r\n"); - trackInfos.Append($"Track id: {e.Track.Id}\r\n"); + trackInfos.Append($"Track kind: {e.Track.kind}\r\n"); + trackInfos.Append($"Track id: {e.Track.id}\r\n"); infoText.text = trackInfos.ToString(); } diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index 3e9247499..3495be428 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -46,10 +46,20 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() { pc2Senders.Add(peer2.AddTrack(e.Track)); }); - foreach (var track in cam.CaptureStream(1280, 720).GetTracks()) + + + cam.CreateRenderStreamTexture(1280, 720); + MediaStream videoStream = new MediaStream(); + int texCount = cam.getStreamTextureCount(); + for (int i = 0; i < texCount; ++i) { - pc1Senders.Add(peer1.AddTrack(track)); + pc1Senders.Add(peer1.AddTrack(new VideoStreamTrack(cam.getStreamTexture(i)))); } + + //foreach (var track in cam.CaptureStream(1280, 720).GetTracks()) + //{ + // pc1Senders.Add(peer1.AddTrack(track)); + //} var conf = new RTCDataChannelInit(true); RTCOfferOptions options1 = default; From b879c1681fe98357d8429d7ca1f14239ffc6bf05 Mon Sep 17 00:00:00 2001 From: koseyile Date: Tue, 2 Jul 2019 05:15:16 +0000 Subject: [PATCH 03/27] Start the name with an uppercase letter. Change the access modifiers from public to internal or protected. --- Assets/Scripts/RenderStreaming.cs | 6 ++-- .../Runtime/Srcipts/MediaStream.cs | 16 ++++----- .../Runtime/Srcipts/MediaStreamTrack.cs | 35 ++++++++++--------- .../Samples/Example/AddMediaStream.cs | 8 ++--- .../Tests/Runtime/MediaStreamTest.cs | 4 +-- 5 files changed, 35 insertions(+), 34 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index 19e97fd7a..d1560d81d 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -62,10 +62,10 @@ public IEnumerator Start() } cam.CreateRenderStreamTexture(1280, 720); videoStream = new MediaStream(); - int texCount = cam.getStreamTextureCount(); + int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - videoStream.AddTrack(new VideoStreamTrack(cam.getStreamTexture(i))); + videoStream.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i))); } @@ -151,7 +151,7 @@ IEnumerator GetOffer() string pattern = @"(a=fmtp:\d+ .*level-asymmetry-allowed=.*)\r\n"; _desc.sdp = Regex.Replace(_desc.sdp, pattern, "$1;x-google-start-bitrate=16000;x-google-max-bitrate=160000\r\n"); pc.SetRemoteDescription(ref _desc); - foreach (var track in videoStream.getTracks()) + foreach (var track in videoStream.GetTracks()) { pc.AddTrack(track); } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index c343c64d7..0082b594e 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -8,8 +8,8 @@ namespace Unity.WebRTC { public class MediaStream { - public IntPtr self; - public string id; + internal IntPtr nativePtr; + protected string id; protected List mediaStreamTrackList = new List(); public MediaStream() : base() @@ -19,10 +19,10 @@ public MediaStream() : base() internal MediaStream(IntPtr ptr) { - self = ptr; - id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); + nativePtr = ptr; + id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(nativePtr)); int trackSize = 0; - IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(self, ref trackSize); + IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(nativePtr, ref trackSize); IntPtr[] tracksPtr = new IntPtr[trackSize]; Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); //TODO: Linux compatibility @@ -50,7 +50,7 @@ public void AddTrack(MediaStreamTrack track) mediaStreamTrackList.Add(track); } - public MediaStreamTrack[] getTracks() + public MediaStreamTrack[] GetTracks() { return mediaStreamTrackList.ToArray(); } @@ -207,12 +207,12 @@ public static class CameraExtension internal static List camCopyRts = new List(); internal static bool started = false; - public static int getStreamTextureCount(this Camera cam) + public static int GetStreamTextureCount(this Camera cam) { return webRTCTextures.Count; } - public static RenderTexture getStreamTexture(this Camera cam, int index) { + public static RenderTexture GetStreamTexture(this Camera cam, int index) { return webRTCTextures[index]; } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index 7b070f0d1..4af54027f 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -6,9 +6,9 @@ namespace Unity.WebRTC { public class MediaStreamTrack { - public IntPtr nativePtr; - public string id; - public TrackKind kind; + internal IntPtr nativePtr; + protected string id; + protected TrackKind kind; public MediaStreamTrack() { @@ -21,20 +21,21 @@ internal MediaStreamTrack(IntPtr ptr) id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(nativePtr)); } - //public bool Enabled - //{ - // get{return NativeMethods.MediaStreamTrackGetEnabled(nativePtr);} - // set{NativeMethods.MediaStreamTrackSetEnabled(nativePtr, value);} - //} - //public TrackState ReadyState - //{ - // get - // {return NativeMethods.MediaStreamTrackGetReadyState(nativePtr);} - // private set { } - //} - - //public TrackKind Kind { get => kind; private set { } } - //public string Id { get => id; private set { } } + public bool Enabled + { + get { return NativeMethods.MediaStreamTrackGetEnabled(nativePtr); } + set { NativeMethods.MediaStreamTrackSetEnabled(nativePtr, value); } + } + + public TrackState ReadyState + { + get + { return NativeMethods.MediaStreamTrackGetReadyState(nativePtr); } + private set { } + } + + public TrackKind Kind { get => kind; private set { } } + public string Id { get => id; private set { } } } public class VideoStreamTrack : MediaStreamTrack diff --git a/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs b/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs index 5e6b609f9..86410f31e 100644 --- a/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs +++ b/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs @@ -159,11 +159,11 @@ void Pc2OnIceCandidate(RTCIceCandidate candidate) } public void AddTracks() { - foreach (var track in audioStream.getTracks()) + foreach (var track in audioStream.GetTracks()) { pc1Senders.Add (pc1.AddTrack(track)); } - foreach(var track in videoStream.getTracks()) + foreach(var track in videoStream.GetTracks()) { pc1Senders.Add(pc1.AddTrack(track)); } @@ -232,8 +232,8 @@ void OnTrack(RTCPeerConnection pc, RTCTrackEvent e) { pc2Senders.Add(pc.AddTrack(e.Track)); trackInfos.Append($"{GetName(pc)} receives remote track:\r\n"); - trackInfos.Append($"Track kind: {e.Track.kind}\r\n"); - trackInfos.Append($"Track id: {e.Track.id}\r\n"); + trackInfos.Append($"Track kind: {e.Track.Kind}\r\n"); + trackInfos.Append($"Track id: {e.Track.Id}\r\n"); infoText.text = trackInfos.ToString(); } diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index 3495be428..5bbb6a7d8 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -50,10 +50,10 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() cam.CreateRenderStreamTexture(1280, 720); MediaStream videoStream = new MediaStream(); - int texCount = cam.getStreamTextureCount(); + int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - pc1Senders.Add(peer1.AddTrack(new VideoStreamTrack(cam.getStreamTexture(i)))); + pc1Senders.Add(peer1.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i)))); } //foreach (var track in cam.CaptureStream(1280, 720).GetTracks()) From 4b296dd2bb5113d7562fb468d138066fc634926a Mon Sep 17 00:00:00 2001 From: koseyile Date: Tue, 2 Jul 2019 06:42:27 +0000 Subject: [PATCH 04/27] remove the temporary comment code. add AudioStreamTrack. --- .../Runtime/Srcipts/MediaStream.cs | 158 ++---------------- .../Runtime/Srcipts/MediaStreamTrack.cs | 62 ++----- .../Samples/Example/AddMediaStream.cs | 11 +- 3 files changed, 37 insertions(+), 194 deletions(-) diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 0082b594e..035bccac1 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -8,8 +8,10 @@ namespace Unity.WebRTC { public class MediaStream { - internal IntPtr nativePtr; - protected string id; + //to do : c++ create two mediastream named "audio" and "vedio". Actually we only need one. + //internal IntPtr nativePtr; + //protected string id; + //public string Id { get => id; private set { } } protected List mediaStreamTrackList = new List(); public MediaStream() : base() @@ -17,26 +19,6 @@ public MediaStream() : base() } - internal MediaStream(IntPtr ptr) - { - nativePtr = ptr; - id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(nativePtr)); - int trackSize = 0; - IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(nativePtr, ref trackSize); - IntPtr[] tracksPtr = new IntPtr[trackSize]; - Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); - //TODO: Linux compatibility - Marshal.FreeCoTaskMem(trackNativePtr); - - for (int i = 0; i < trackSize; i++) - { - MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); - //track.stopTrack += StopTrack; - //track.getRts += GetRts; - mediaStreamTrackList.Add(track); - } - } - public MediaStream(MediaStreamTrack[] tracks) : base() { foreach (var t in tracks) @@ -55,123 +37,7 @@ public MediaStreamTrack[] GetTracks() return mediaStreamTrackList.ToArray(); } } - - //public class MediaStream - //{ - // private IntPtr self; - // private string id; - // public string Id { get => id; private set { } } - - // private Dictionary VideoTrackToRts; - // private List AudioTracks; - - // //private void StopTrack(MediaStreamTrack track) - // //{ - - // // if (track.kind == TrackKind.Video) - // // { - // // NativeMethods.StopMediaStreamTrack(track.nativePtr); - // // RenderTexture[] rts = VideoTrackToRts[track]; - // // if (rts != null) - // // { - // // CameraExtension.RemoveRt(rts); - // // rts[0].Release(); - // // rts[1].Release(); - // // UnityEngine.Object.Destroy(rts[0]); - // // UnityEngine.Object.Destroy(rts[1]); - // // } - // // } - // // else - // // { - // // Audio.Stop(); - // // } - - // //} - // private RenderTexture[] GetRts(MediaStreamTrack track) - // { - // return VideoTrackToRts[track]; - // } - // public MediaStreamTrack[] GetTracks() - // { - // MediaStreamTrack[] tracks = new MediaStreamTrack[VideoTrackToRts.Keys.Count + AudioTracks.Count]; - // AudioTracks.CopyTo(tracks, 0); - // VideoTrackToRts.Keys.CopyTo(tracks, AudioTracks.Count); - // return tracks; - // } - // //public MediaStreamTrack[] GetAudioTracks() - // //{ - // // return AudioTracks.ToArray(); - // //} - // //public MediaStreamTrack[] GetVideoTracks() - // //{ - // // MediaStreamTrack[] tracks = new MediaStreamTrack[VideoTrackToRts.Keys.Count]; - // // VideoTrackToRts.Keys.CopyTo(tracks, 0); - // // return tracks; - // //} - - // //public void AddTrack(MediaStreamTrack track) - // //{ - // // if(track.kind == TrackKind.Video) - // // { - // // VideoTrackToRts[track] = track.getRts(track); - // // } - // // else - // // { - // // AudioTracks.Add(track); - // // } - // // NativeMethods.MediaStreamAddTrack(self, track.self); - // //} - // //public void RemoveTrack(MediaStreamTrack track) - // //{ - // // NativeMethods.MediaStreamRemoveTrack(self, track.self); - // //} - // //for camera CaptureStream - // internal MediaStream(RenderTexture[] rts, IntPtr ptr) - // { - // self = ptr; - // id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); - // VideoTrackToRts = new Dictionary(); - // AudioTracks = new List(); - // //get initial tracks - // int trackSize = 0; - // IntPtr tracksNativePtr = NativeMethods.MediaStreamGetVideoTracks(self, ref trackSize); - // IntPtr[] tracksPtr = new IntPtr[trackSize]; - // Marshal.Copy(tracksNativePtr, tracksPtr, 0, trackSize); - // //TODO: Linux compatibility - // Marshal.FreeCoTaskMem(tracksNativePtr); - // for (int i = 0; i < trackSize; i++) - // { - // MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); - // //track.stopTrack += StopTrack; - // //track.getRts += GetRts; - // VideoTrackToRts[track] = rts; - // } - // } - // //for audio CaptureStream - // internal MediaStream(IntPtr ptr) - // { - // self = ptr; - // id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(self)); - // VideoTrackToRts = new Dictionary(); - // AudioTracks = new List(); - // //get initial tracks - // int trackSize = 0; - // IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(self, ref trackSize); - // IntPtr[] tracksPtr = new IntPtr[trackSize]; - // Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); - // //TODO: Linux compatibility - // Marshal.FreeCoTaskMem(trackNativePtr); - - // for (int i = 0; i < trackSize; i++) - // { - // MediaStreamTrack track = new MediaStreamTrack(tracksPtr[i]); - // //track.stopTrack += StopTrack; - // //track.getRts += GetRts; - // AudioTracks.Add(track); - // } - // } - - //} + internal class Cleaner : MonoBehaviour { private Action onDestroy; @@ -250,14 +116,6 @@ public static void CreateRenderStreamTexture(this Camera cam, int width, int hei started = true; } - public static MediaStream CaptureStream(this Camera cam, int width, int height) - { - cam.CreateRenderStreamTexture(width, height); - - int textureIndex = 0; - int rtcMipLevel = (int)Math.Pow(2, textureIndex); //1 2 4 8 - return new MediaStream(WebRTC.Context.CaptureVideoStream(webRTCTextures[textureIndex].GetNativeTexturePtr(), width/rtcMipLevel, height/rtcMipLevel)); - } public static void RemoveRt(RenderTexture[] rts) { camCopyRts.Remove(rts); @@ -277,7 +135,11 @@ public static MediaStream CaptureStream() { audioInput.BeginRecording(); started = true; - return new MediaStream(WebRTC.Context.CaptureAudioStream()); + + MediaStream mediaStream = new MediaStream(); + AudioStreamTrack audioStreamTrack = new AudioStreamTrack(); + mediaStream.AddTrack(audioStreamTrack); + return mediaStream; } public static void Update() { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index 4af54027f..5be379d56 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -58,54 +58,26 @@ public VideoStreamTrack(RenderTexture rt) : base() } } + public class AudioStreamTrack : MediaStreamTrack + { + public AudioStreamTrack() : base() + { + IntPtr nativeAudioStreamPtr = WebRTC.Context.CaptureAudioStream(); + string nativeAudioStreamID = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(nativeAudioStreamPtr)); + int trackSize = 0; + IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(nativeAudioStreamPtr, ref trackSize); + IntPtr[] tracksPtr = new IntPtr[trackSize]; + Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); + //TODO: Linux compatibility + Marshal.FreeCoTaskMem(trackNativePtr); + nativePtr = tracksPtr[0]; + kind = NativeMethods.MediaStreamTrackGetKind(nativePtr); + id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(nativePtr)); + } + } - //public class MediaStreamTrack - //{ - // internal IntPtr self; - // private TrackKind kind; - // private string id; - // private bool enabled; - // private TrackState readyState; - // internal Action stopTrack; - // internal Func getRts; - - // public bool Enabled - // { - // get - // { - // return NativeMethods.MediaStreamTrackGetEnabled(self); - // } - // set - // { - // NativeMethods.MediaStreamTrackSetEnabled(self, value); - // } - // } - // public TrackState ReadyState - // { - // get - // { - // return NativeMethods.MediaStreamTrackGetReadyState(self); - // } - // private set { } - // } - - // public TrackKind Kind { get => kind; private set { } } - // public string Id { get => id; private set { } } - - // internal MediaStreamTrack(IntPtr ptr) - // { - // self = ptr; - // kind = NativeMethods.MediaStreamTrackGetKind(self); - // id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(self)); - // } - // //Disassociate track from its source(video or audio), not for destroying the track - // public void Stop() - // { - // stopTrack(this); - // } - //} public enum TrackKind { diff --git a/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs b/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs index 86410f31e..78298f466 100644 --- a/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs +++ b/Packages/com.unity.webrtc/Samples/Example/AddMediaStream.cs @@ -213,7 +213,16 @@ void Call() RTCDataChannelInit conf = new RTCDataChannelInit(true); dataChannel = pc1.CreateDataChannel("data", ref conf); audioStream = Audio.CaptureStream(); - videoStream = cam.CaptureStream(1280, 720); + + cam.CreateRenderStreamTexture(1280, 720); + videoStream = new MediaStream(); + int texCount = cam.GetStreamTextureCount(); + for (int i = 0; i < texCount; ++i) + { + videoStream.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i))); + } + + RtImage.texture = cam.targetTexture; } From e924c1da1fccf10da429dc0b2a93065b8e6bd476 Mon Sep 17 00:00:00 2001 From: koseyile Date: Tue, 2 Jul 2019 07:47:24 +0000 Subject: [PATCH 05/27] remove unused comments. public modify to internal. --- Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs | 3 --- Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs | 2 +- Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs | 4 ---- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 035bccac1..89db2ab41 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -9,9 +9,6 @@ namespace Unity.WebRTC public class MediaStream { //to do : c++ create two mediastream named "audio" and "vedio". Actually we only need one. - //internal IntPtr nativePtr; - //protected string id; - //public string Id { get => id; private set { } } protected List mediaStreamTrackList = new List(); public MediaStream() : base() diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index 5be379d56..b7b7466c6 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -9,7 +9,7 @@ public class MediaStreamTrack internal IntPtr nativePtr; protected string id; protected TrackKind kind; - public MediaStreamTrack() + internal MediaStreamTrack() { } diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index 5bbb6a7d8..bbe9db942 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -56,10 +56,6 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() pc1Senders.Add(peer1.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i)))); } - //foreach (var track in cam.CaptureStream(1280, 720).GetTracks()) - //{ - // pc1Senders.Add(peer1.AddTrack(track)); - //} var conf = new RTCDataChannelInit(true); RTCOfferOptions options1 = default; From 2e903d047bb74d05c43de04df71eb74749bec81c Mon Sep 17 00:00:00 2001 From: chenyuan Date: Tue, 9 Jul 2019 11:51:52 +0800 Subject: [PATCH 06/27] add new method "createVideoTrack" to Context class. --- Plugin/WebRTCPlugin/Context.cpp | 41 +++++++++++++---------- Plugin/WebRTCPlugin/Context.h | 10 +++--- Plugin/WebRTCPlugin/DummyVideoEncoder.cpp | 7 +++- Plugin/WebRTCPlugin/DummyVideoEncoder.h | 4 ++- Plugin/WebRTCPlugin/WebRTCPlugin.cpp | 4 +-- 5 files changed, 40 insertions(+), 26 deletions(-) diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index 59c5262dd..9f42536b7 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -292,9 +292,10 @@ namespace WebRTC rtc::InitializeSSL(); audioDevice = new rtc::RefCountedObject(); - nvVideoCapturerUnique = std::make_unique(); - nvVideoCapturer = nvVideoCapturerUnique.get(); - auto dummyVideoEncoderFactory = std::make_unique(nvVideoCapturer); + //nvVideoCapturerUnique = std::make_unique(); + //nvVideoCapturer = nvVideoCapturerUnique.get(); + auto dummyVideoEncoderFactory = std::make_unique(); + pDummyVideoEncoderFactory = dummyVideoEncoderFactory.get(); peerConnectionFactory = webrtc::CreatePeerConnectionFactory( workerThread.get(), @@ -314,9 +315,8 @@ namespace WebRTC clients.clear(); peerConnectionFactory = nullptr; audioTrack = nullptr; - videoTracks.clear(); audioStream = nullptr; - videoStreams.clear(); + mediaStream = nullptr; workerThread->Quit(); workerThread.reset(); @@ -324,21 +324,28 @@ namespace WebRTC signalingThread.reset(); } - webrtc::MediaStreamInterface* Context::CreateVideoStream(UnityFrameBuffer* frameBuffer) + webrtc::MediaStreamInterface* Context::CreateVideoStream(UnityFrameBuffer* frameBuffer, int32 width, int32 height) { - //TODO: label and stream id should be maintained in some way for multi-stream - auto videoTrack = peerConnectionFactory->CreateVideoTrack( - "video", peerConnectionFactory->CreateVideoSource(std::move(nvVideoCapturerUnique))); - if (!videoTracks.count(frameBuffer)) - { - videoTracks[frameBuffer] = videoTrack; - } - auto videoStream = peerConnectionFactory->CreateLocalMediaStream("video"); - videoStream->AddTrack(videoTrack); - videoStreams.push_back(videoStream); + ////TODO: label and stream id should be maintained in some way for multi-stream + //create track + auto videoTrack = CreateVideoTrack("video", frameBuffer, width, height); + //create stream + mediaStream = peerConnectionFactory->CreateLocalMediaStream("video"); + mediaStream->AddTrack(videoTrack); + return mediaStream.get(); + } + + rtc::scoped_refptr Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) + { + nvVideoCapturerUnique = std::make_unique(); + nvVideoCapturer = nvVideoCapturerUnique.get(); + nvVideoCapturer->InitializeEncoder(width, height); + pDummyVideoEncoderFactory->SetCapturer(nvVideoCapturer); + auto videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(std::move(nvVideoCapturerUnique))); nvVideoCapturer->unityRT = frameBuffer; nvVideoCapturer->StartEncoder(); - return videoStream.get(); + + return videoTrack; } webrtc::MediaStreamInterface* Context::CreateAudioStream() diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index 939e09574..899de2807 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -42,13 +42,13 @@ namespace WebRTC { public: explicit Context(int uid = -1); - webrtc::MediaStreamInterface* CreateVideoStream(UnityFrameBuffer* frameBuffer); + webrtc::MediaStreamInterface* CreateVideoStream(UnityFrameBuffer* frameBuffer, int32 width, int32 height); + rtc::scoped_refptr CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height); webrtc::MediaStreamInterface* CreateAudioStream(); ~Context(); PeerConnectionObject* CreatePeerConnection(int id); PeerConnectionObject* CreatePeerConnection(int id, const std::string& conf); - void InitializeEncoder(int32 width, int32 height) { nvVideoCapturer->InitializeEncoder(width, height); } void EncodeFrame() { nvVideoCapturer->EncodeVideoData(); } void StopCapturer() { nvVideoCapturer->Stop(); } void ProcessAudioData(const float* data, int32 size) { audioDevice->ProcessAudioData(data, size); } @@ -59,14 +59,14 @@ namespace WebRTC std::unique_ptr signalingThread; std::map> clients; rtc::scoped_refptr peerConnectionFactory; + DummyVideoEncoderFactory* pDummyVideoEncoderFactory; + rtc::scoped_refptr mediaStream; + NvVideoCapturer* nvVideoCapturer; std::unique_ptr nvVideoCapturerUnique; rtc::scoped_refptr audioDevice; rtc::scoped_refptr audioTrack; rtc::scoped_refptr audioStream; - //TODO: move videoTrack to NvVideoCapturer and maintain multiple NvVideoCapturer here - std::vector> videoStreams; - std::map> videoTracks; }; class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp index 1a5dff007..e20840a76 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp @@ -67,7 +67,12 @@ namespace WebRTC SetRate(allocation.get_sum_kbps() * 1000); return 0; } - DummyVideoEncoderFactory::DummyVideoEncoderFactory(NvVideoCapturer* videoCapturer):capturer(videoCapturer){} + + DummyVideoEncoderFactory::DummyVideoEncoderFactory() + { + + } + std::vector DummyVideoEncoderFactory::GetSupportedFormats() const { const absl::optional profileLevelId = diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.h b/Plugin/WebRTCPlugin/DummyVideoEncoder.h index 43437e2ad..ad8f81601 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.h +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.h @@ -51,7 +51,9 @@ namespace WebRTC // Creates a VideoEncoder for the specified format. virtual std::unique_ptr CreateVideoEncoder( const webrtc::SdpVideoFormat& format) override; - DummyVideoEncoderFactory(NvVideoCapturer* videoCapturer); + DummyVideoEncoderFactory(); + + void SetCapturer(NvVideoCapturer* _capturer) { capturer = _capturer; } private: NvVideoCapturer* capturer; }; diff --git a/Plugin/WebRTCPlugin/WebRTCPlugin.cpp b/Plugin/WebRTCPlugin/WebRTCPlugin.cpp index 9ce258472..a9144b74b 100644 --- a/Plugin/WebRTCPlugin/WebRTCPlugin.cpp +++ b/Plugin/WebRTCPlugin/WebRTCPlugin.cpp @@ -31,8 +31,8 @@ extern "C" { UNITY_INTERFACE_EXPORT webrtc::MediaStreamInterface* CaptureVideoStream(Context* context, UnityFrameBuffer* rt, int32 width, int32 height) { - context->InitializeEncoder(width, height); - return context->CreateVideoStream(rt); + //context->InitializeEncoder(width, height); + return context->CreateVideoStream(rt, width, height); } //TODO: Multi-track support UNITY_INTERFACE_EXPORT void StopMediaStreamTrack(webrtc::MediaStreamTrackInterface* track) From 936991a4c348172013b2e981d0e344d5b4a671aa Mon Sep 17 00:00:00 2001 From: chenyuan Date: Tue, 9 Jul 2019 16:43:01 +0800 Subject: [PATCH 07/27] video and audio share one mediaStream. --- Plugin/WebRTCPlugin/Context.cpp | 22 +++++++++++----------- Plugin/WebRTCPlugin/Context.h | 3 +-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index 9f42536b7..6c31de412 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -292,8 +292,6 @@ namespace WebRTC rtc::InitializeSSL(); audioDevice = new rtc::RefCountedObject(); - //nvVideoCapturerUnique = std::make_unique(); - //nvVideoCapturer = nvVideoCapturerUnique.get(); auto dummyVideoEncoderFactory = std::make_unique(); pDummyVideoEncoderFactory = dummyVideoEncoderFactory.get(); @@ -308,14 +306,14 @@ namespace WebRTC webrtc::CreateBuiltinVideoDecoderFactory(), nullptr, nullptr); + + mediaStream = peerConnectionFactory->CreateLocalMediaStream("mediaStream"); } Context::~Context() { clients.clear(); peerConnectionFactory = nullptr; - audioTrack = nullptr; - audioStream = nullptr; mediaStream = nullptr; workerThread->Quit(); @@ -327,10 +325,7 @@ namespace WebRTC webrtc::MediaStreamInterface* Context::CreateVideoStream(UnityFrameBuffer* frameBuffer, int32 width, int32 height) { ////TODO: label and stream id should be maintained in some way for multi-stream - //create track auto videoTrack = CreateVideoTrack("video", frameBuffer, width, height); - //create stream - mediaStream = peerConnectionFactory->CreateLocalMediaStream("video"); mediaStream->AddTrack(videoTrack); return mediaStream.get(); } @@ -349,6 +344,13 @@ namespace WebRTC } webrtc::MediaStreamInterface* Context::CreateAudioStream() + { + auto audioTrack = CreateAudioTrack(); + mediaStream->AddTrack(audioTrack); + return mediaStream.get(); + } + + rtc::scoped_refptr Context::CreateAudioTrack() { //avoid optimization specially for voice cricket::AudioOptions audioOptions; @@ -356,10 +358,8 @@ namespace WebRTC audioOptions.noise_suppression = false; audioOptions.highpass_filter = false; //TODO: label and stream id should be maintained in some way for multi-stream - audioTrack = peerConnectionFactory->CreateAudioTrack("audio", peerConnectionFactory->CreateAudioSource(audioOptions)); - audioStream = peerConnectionFactory->CreateLocalMediaStream("audio"); - audioStream->AddTrack(audioTrack); - return audioStream.get(); + auto audioTrack = peerConnectionFactory->CreateAudioTrack("audio", peerConnectionFactory->CreateAudioSource(audioOptions)); + return audioTrack; } PeerSDPObserver* PeerSDPObserver::Create(DelegateSetSDSuccess onSuccess, DelegateSetSDFailure onFailure) diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index 899de2807..be2828460 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -45,6 +45,7 @@ namespace WebRTC webrtc::MediaStreamInterface* CreateVideoStream(UnityFrameBuffer* frameBuffer, int32 width, int32 height); rtc::scoped_refptr CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height); webrtc::MediaStreamInterface* CreateAudioStream(); + rtc::scoped_refptr CreateAudioTrack(); ~Context(); PeerConnectionObject* CreatePeerConnection(int id); @@ -65,8 +66,6 @@ namespace WebRTC NvVideoCapturer* nvVideoCapturer; std::unique_ptr nvVideoCapturerUnique; rtc::scoped_refptr audioDevice; - rtc::scoped_refptr audioTrack; - rtc::scoped_refptr audioStream; }; class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver From f3972bd7d1d59a5c16f9adba270c4c4539edf4ed Mon Sep 17 00:00:00 2001 From: chenyuan Date: Tue, 9 Jul 2019 19:26:33 +0800 Subject: [PATCH 08/27] change name "videostream" to "mediastream" and add audio track to media stream --- Assets/Scripts/RenderStreaming.cs | 17 +++++++++++------ .../Runtime/Srcipts/MediaStream.cs | 8 ++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index c7044173e..f5187efaa 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -41,7 +41,7 @@ public class RenderStreaming : MonoBehaviour private Dictionary> mapChannels = new Dictionary>(); private RTCConfiguration conf; private string sessionId; - private MediaStream videoStream; + private MediaStream mediaStream; public void Awake() { @@ -52,6 +52,7 @@ public void Awake() public void OnDestroy() { + Audio.Stop(); WebRTC.WebRTC.Finalize(); } public IEnumerator Start() @@ -61,14 +62,17 @@ public IEnumerator Start() yield break; } - cam.CreateRenderStreamTexture(1280, 720); - videoStream = new MediaStream(); - int texCount = cam.GetStreamTextureCount(); + captureCamera.CreateRenderStreamTexture(1280, 720); + mediaStream = new MediaStream(); + int texCount = captureCamera.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - videoStream.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i))); + mediaStream.AddTrack(new VideoStreamTrack(captureCamera.GetStreamTexture(i))); } + mediaStream.AddTrack(new AudioStreamTrack()); + Audio.Start(); + signaling = new Signaling(urlSignaling); var opCreate = signaling.Create(); yield return opCreate; @@ -150,10 +154,11 @@ IEnumerator GetOffer() string pattern = @"(a=fmtp:\d+ .*level-asymmetry-allowed=.*)\r\n"; _desc.sdp = Regex.Replace(_desc.sdp, pattern, "$1;x-google-start-bitrate=16000;x-google-max-bitrate=160000\r\n"); pc.SetRemoteDescription(ref _desc); - foreach (var track in videoStream.GetTracks()) + foreach (var track in mediaStream.GetTracks()) { pc.AddTrack(track); } + StartCoroutine(Answer(connectionId)); } } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 89db2ab41..052106344 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -138,6 +138,7 @@ public static MediaStream CaptureStream() mediaStream.AddTrack(audioStreamTrack); return mediaStream; } + public static void Update() { if (started) @@ -145,6 +146,13 @@ public static void Update() audioInput.UpdateAudio(); } } + + public static void Start() + { + audioInput.BeginRecording(); + started = true; + } + public static void Stop() { if (started) From e400b81d0122b9c424d71a874b7c42cfbe06d8c7 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Thu, 11 Jul 2019 19:27:39 +0800 Subject: [PATCH 09/27] decoupling streams and tracks. delete these functions: CaptureVideoStream CaptureAudioStream --- Assets/Scripts/RenderStreaming.cs | 2 +- .../Runtime/Srcipts/MediaStream.cs | 23 +++++------- .../Runtime/Srcipts/MediaStreamTrack.cs | 34 ++---------------- .../Runtime/Srcipts/WebRTC.cs | 12 ++++--- .../Samples/Example/MediaStreamSample.cs | 16 ++++----- .../Tests/Runtime/MediaStreamTest.cs | 6 ++-- Plugin/WebRTCPlugin/Context.cpp | 35 +++++++++---------- Plugin/WebRTCPlugin/Context.h | 12 ++++--- Plugin/WebRTCPlugin/WebRTCPlugin.cpp | 22 +++++++----- 9 files changed, 68 insertions(+), 94 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index f5187efaa..085c8f108 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -63,7 +63,7 @@ public IEnumerator Start() } captureCamera.CreateRenderStreamTexture(1280, 720); - mediaStream = new MediaStream(); + mediaStream = new MediaStream("MediaStream"); int texCount = captureCamera.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 052106344..5d82eca07 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -8,24 +8,27 @@ namespace Unity.WebRTC { public class MediaStream { - //to do : c++ create two mediastream named "audio" and "vedio". Actually we only need one. + internal IntPtr nativePtr; protected List mediaStreamTrackList = new List(); - public MediaStream() : base() + public MediaStream(string label) : base() { - + nativePtr = WebRTC.Context.CreateMediaStream(label); } - public MediaStream(MediaStreamTrack[] tracks) : base() + public MediaStream(string label, MediaStreamTrack[] tracks) : base() { + nativePtr = WebRTC.Context.CreateMediaStream(label); + foreach (var t in tracks) { - mediaStreamTrackList.Add(t); + AddTrack(t); } } public void AddTrack(MediaStreamTrack track) { + NativeMethods.MediaStreamAddTrack(nativePtr, track.nativePtr); mediaStreamTrackList.Add(track); } @@ -128,16 +131,6 @@ public static class Audio { private static bool started = false; private static AudioInput audioInput = new AudioInput(); - public static MediaStream CaptureStream() - { - audioInput.BeginRecording(); - started = true; - - MediaStream mediaStream = new MediaStream(); - AudioStreamTrack audioStreamTrack = new AudioStreamTrack(); - mediaStream.AddTrack(audioStreamTrack); - return mediaStream; - } public static void Update() { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index b7b7466c6..25a716ede 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -9,10 +9,6 @@ public class MediaStreamTrack internal IntPtr nativePtr; protected string id; protected TrackKind kind; - internal MediaStreamTrack() - { - - } internal MediaStreamTrack(IntPtr ptr) { @@ -40,41 +36,15 @@ private set { } public class VideoStreamTrack : MediaStreamTrack { - public VideoStreamTrack(RenderTexture rt) : base() + public VideoStreamTrack(RenderTexture rt) : base(WebRTC.Context.CreateVideoTrack("videoTrack", rt.GetNativeTexturePtr(), rt.width, rt.height)) { - IntPtr nativeVideoStreamPtr = WebRTC.Context.CaptureVideoStream(rt.GetNativeTexturePtr(), rt.width, rt.height); - string nativeVideoStreamID = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(nativeVideoStreamPtr)); - - int trackSize = 0; - IntPtr tracksNativePtr = NativeMethods.MediaStreamGetVideoTracks(nativeVideoStreamPtr, ref trackSize); - IntPtr[] tracksPtr = new IntPtr[trackSize]; - Marshal.Copy(tracksNativePtr, tracksPtr, 0, trackSize); - Marshal.FreeCoTaskMem(tracksNativePtr); - - nativePtr = tracksPtr[0]; - kind = NativeMethods.MediaStreamTrackGetKind(nativePtr); - id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(nativePtr)); - } } public class AudioStreamTrack : MediaStreamTrack { - public AudioStreamTrack() : base() + public AudioStreamTrack() : base(WebRTC.Context.CreateAudioTrack("audioTrack")) { - IntPtr nativeAudioStreamPtr = WebRTC.Context.CaptureAudioStream(); - string nativeAudioStreamID = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(nativeAudioStreamPtr)); - - int trackSize = 0; - IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(nativeAudioStreamPtr, ref trackSize); - IntPtr[] tracksPtr = new IntPtr[trackSize]; - Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize); - //TODO: Linux compatibility - Marshal.FreeCoTaskMem(trackNativePtr); - - nativePtr = tracksPtr[0]; - kind = NativeMethods.MediaStreamTrackGetKind(nativePtr); - id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(nativePtr)); } } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs index 0b1caadf3..5ea60fa1b 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs @@ -362,7 +362,11 @@ internal static class NativeMethods [DllImport(WebRTC.Lib)] public static extern IntPtr CaptureVideoStream(IntPtr context, IntPtr rt, int width, int height); [DllImport(WebRTC.Lib)] - public static extern IntPtr CaptureAudioStream(IntPtr context); + public static extern IntPtr CreateMediaStream(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label); + [DllImport(WebRTC.Lib)] + public static extern IntPtr CreateVideoTrack(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label, IntPtr rt, int width, int height); + [DllImport(WebRTC.Lib)] + public static extern IntPtr CreateAudioTrack(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label); [DllImport(WebRTC.Lib)] public static extern void MediaStreamAddTrack(IntPtr stream, IntPtr track); [DllImport(WebRTC.Lib)] @@ -399,9 +403,9 @@ internal struct Context public static bool ToBool(Context v) { return v; } public static Context Create(int uid = 0) { return NativeMethods.ContextCreate(uid); } public void Destroy(int uid = 0) { NativeMethods.ContextDestroy(uid); self = IntPtr.Zero; } - public IntPtr CaptureVideoStream(IntPtr rt, int width, int height) { return NativeMethods.CaptureVideoStream(self, rt, width, height); } - public IntPtr CaptureAudioStream() { return NativeMethods.CaptureAudioStream(self); } - + public IntPtr CreateMediaStream(string label) { return NativeMethods.CreateMediaStream(self, label); } + public IntPtr CreateVideoTrack(string label, IntPtr rt, int width, int height) { return NativeMethods.CreateVideoTrack(self, label, rt, width, height); } + public IntPtr CreateAudioTrack(string label) {return NativeMethods.CreateAudioTrack(self, label);} } } diff --git a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs index 6a017732b..dc28c0d66 100644 --- a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs +++ b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs @@ -19,7 +19,7 @@ public class MediaStreamSample : MonoBehaviour private RTCPeerConnection pc1, pc2; private List pc1Senders, pc2Senders; - private Unity.WebRTC.MediaStream audioStream, videoStream; + private Unity.WebRTC.MediaStream mediaStream; private RTCDataChannel dataChannel, remoteDataChannel; private Coroutine sdpCheck; private string msg; @@ -159,14 +159,11 @@ void Pc2OnIceCandidate(RTCIceCandidate candidate) } public void AddTracks() { - foreach (var track in audioStream.GetTracks()) + foreach (var track in mediaStream.GetTracks()) { pc1Senders.Add (pc1.AddTrack(track)); } - foreach(var track in videoStream.GetTracks()) - { - pc1Senders.Add(pc1.AddTrack(track)); - } + if(!videoUpdateStarted) { StartCoroutine(WebRTC.Update()); @@ -212,16 +209,17 @@ void Call() RTCDataChannelInit conf = new RTCDataChannelInit(true); dataChannel = pc1.CreateDataChannel("data", ref conf); - audioStream = Audio.CaptureStream(); cam.CreateRenderStreamTexture(1280, 720); - videoStream = new MediaStream(); + mediaStream = new MediaStream("mediaStream"); int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - videoStream.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i))); + mediaStream.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i))); } + mediaStream.AddTrack(new AudioStreamTrack()); + Audio.Start(); RtImage.texture = cam.targetTexture; diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index bbe9db942..ee986c32f 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -49,11 +49,13 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() cam.CreateRenderStreamTexture(1280, 720); - MediaStream videoStream = new MediaStream(); + MediaStream mediaStream = new MediaStream("mediaStream"); int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - pc1Senders.Add(peer1.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i)))); + VideoStreamTrack videoStreamTrack = new VideoStreamTrack(cam.GetStreamTexture(i)); + mediaStream.AddTrack(videoStreamTrack); + pc1Senders.Add(peer1.AddTrack(videoStreamTrack)); } var conf = new RTCDataChannelInit(true); diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index 6c31de412..c4c28dedd 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -307,14 +307,17 @@ namespace WebRTC nullptr, nullptr); - mediaStream = peerConnectionFactory->CreateLocalMediaStream("mediaStream"); } Context::~Context() { clients.clear(); peerConnectionFactory = nullptr; - mediaStream = nullptr; + + videoTrack = nullptr; + audioTrack = nullptr; + + mediaStreamMap.clear(); workerThread->Quit(); workerThread.reset(); @@ -322,35 +325,31 @@ namespace WebRTC signalingThread.reset(); } - webrtc::MediaStreamInterface* Context::CreateVideoStream(UnityFrameBuffer* frameBuffer, int32 width, int32 height) + webrtc::MediaStreamInterface* Context::CreateMediaStream(const std::string& stream_id) { - ////TODO: label and stream id should be maintained in some way for multi-stream - auto videoTrack = CreateVideoTrack("video", frameBuffer, width, height); - mediaStream->AddTrack(videoTrack); - return mediaStream.get(); + if (mediaStreamMap.count(stream_id) == 0) + { + mediaStreamMap[stream_id] = peerConnectionFactory->CreateLocalMediaStream(stream_id); + } + + return mediaStreamMap[stream_id]; } - rtc::scoped_refptr Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) + webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) { nvVideoCapturerUnique = std::make_unique(); nvVideoCapturer = nvVideoCapturerUnique.get(); nvVideoCapturer->InitializeEncoder(width, height); pDummyVideoEncoderFactory->SetCapturer(nvVideoCapturer); - auto videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(std::move(nvVideoCapturerUnique))); + + videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(std::move(nvVideoCapturerUnique))); nvVideoCapturer->unityRT = frameBuffer; nvVideoCapturer->StartEncoder(); return videoTrack; } - webrtc::MediaStreamInterface* Context::CreateAudioStream() - { - auto audioTrack = CreateAudioTrack(); - mediaStream->AddTrack(audioTrack); - return mediaStream.get(); - } - - rtc::scoped_refptr Context::CreateAudioTrack() + webrtc::MediaStreamTrackInterface* Context::CreateAudioTrack(const std::string& label) { //avoid optimization specially for voice cricket::AudioOptions audioOptions; @@ -358,7 +357,7 @@ namespace WebRTC audioOptions.noise_suppression = false; audioOptions.highpass_filter = false; //TODO: label and stream id should be maintained in some way for multi-stream - auto audioTrack = peerConnectionFactory->CreateAudioTrack("audio", peerConnectionFactory->CreateAudioSource(audioOptions)); + audioTrack = peerConnectionFactory->CreateAudioTrack(label, peerConnectionFactory->CreateAudioSource(audioOptions)); return audioTrack; } diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index be2828460..6ba184b34 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -42,10 +42,9 @@ namespace WebRTC { public: explicit Context(int uid = -1); - webrtc::MediaStreamInterface* CreateVideoStream(UnityFrameBuffer* frameBuffer, int32 width, int32 height); - rtc::scoped_refptr CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height); - webrtc::MediaStreamInterface* CreateAudioStream(); - rtc::scoped_refptr CreateAudioTrack(); + webrtc::MediaStreamInterface* CreateMediaStream(const std::string& stream_id); + webrtc::MediaStreamTrackInterface* CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height); + webrtc::MediaStreamTrackInterface* CreateAudioTrack(const std::string& label); ~Context(); PeerConnectionObject* CreatePeerConnection(int id); @@ -61,11 +60,14 @@ namespace WebRTC std::map> clients; rtc::scoped_refptr peerConnectionFactory; DummyVideoEncoderFactory* pDummyVideoEncoderFactory; - rtc::scoped_refptr mediaStream; + std::map> mediaStreamMap; NvVideoCapturer* nvVideoCapturer; std::unique_ptr nvVideoCapturerUnique; rtc::scoped_refptr audioDevice; + + rtc::scoped_refptr videoTrack; + rtc::scoped_refptr audioTrack; }; class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver diff --git a/Plugin/WebRTCPlugin/WebRTCPlugin.cpp b/Plugin/WebRTCPlugin/WebRTCPlugin.cpp index a9144b74b..707c0d88d 100644 --- a/Plugin/WebRTCPlugin/WebRTCPlugin.cpp +++ b/Plugin/WebRTCPlugin/WebRTCPlugin.cpp @@ -29,11 +29,21 @@ namespace WebRTC extern "C" { - UNITY_INTERFACE_EXPORT webrtc::MediaStreamInterface* CaptureVideoStream(Context* context, UnityFrameBuffer* rt, int32 width, int32 height) + UNITY_INTERFACE_EXPORT webrtc::MediaStreamInterface* CreateMediaStream(Context* context, const char* label) { - //context->InitializeEncoder(width, height); - return context->CreateVideoStream(rt, width, height); + return context->CreateMediaStream(label); } + + UNITY_INTERFACE_EXPORT webrtc::MediaStreamTrackInterface* CreateVideoTrack(Context* context, const char* label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) + { + return context->CreateVideoTrack(label, frameBuffer, width, height); + } + + UNITY_INTERFACE_EXPORT webrtc::MediaStreamTrackInterface* CreateAudioTrack(Context* context, const char* label) + { + return context->CreateAudioTrack(label); + } + //TODO: Multi-track support UNITY_INTERFACE_EXPORT void StopMediaStreamTrack(webrtc::MediaStreamTrackInterface* track) { @@ -45,11 +55,6 @@ extern "C" return ContextManager::GetNvEncSupported(); } - UNITY_INTERFACE_EXPORT webrtc::MediaStreamInterface* CaptureAudioStream(Context* context) - { - return context->CreateAudioStream(); - } - UNITY_INTERFACE_EXPORT void MediaStreamAddTrack(webrtc::MediaStreamInterface* stream, webrtc::MediaStreamTrackInterface* track) { if (track->kind() == "audio") @@ -61,6 +66,7 @@ extern "C" stream->AddTrack((webrtc::VideoTrackInterface*)track); } } + UNITY_INTERFACE_EXPORT void MediaStreamRemoveTrack(webrtc::MediaStreamInterface* stream, webrtc::MediaStreamTrackInterface* track) { if (track->kind() == "audio") From 7eedfb7c5e6029851f98c6c7b86c702a129f62da Mon Sep 17 00:00:00 2001 From: chenyuan Date: Fri, 12 Jul 2019 18:46:43 +0800 Subject: [PATCH 10/27] make DummyVideoEncoder support multiple capturer. --- Plugin/WebRTCPlugin/Context.cpp | 39 ++++++++++++++++------- Plugin/WebRTCPlugin/Context.h | 11 +++---- Plugin/WebRTCPlugin/DummyVideoEncoder.cpp | 11 +++++-- Plugin/WebRTCPlugin/DummyVideoEncoder.h | 4 +-- 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index c4c28dedd..593d7e800 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -314,10 +314,9 @@ namespace WebRTC clients.clear(); peerConnectionFactory = nullptr; - videoTrack = nullptr; - audioTrack = nullptr; - + mediaSteamTrackList.clear(); mediaStreamMap.clear(); + nvVideoCapturerList.clear(); workerThread->Quit(); workerThread.reset(); @@ -325,6 +324,22 @@ namespace WebRTC signalingThread.reset(); } + void Context::EncodeFrame() + { + for (std::list::iterator it= nvVideoCapturerList.begin(); it!= nvVideoCapturerList.end(); ++it) + { + (*it)->EncodeVideoData(); + } + } + + void Context::StopCapturer() + { + for (std::list::iterator it = nvVideoCapturerList.begin(); it != nvVideoCapturerList.end(); ++it) + { + (*it)->Stop(); + } + } + webrtc::MediaStreamInterface* Context::CreateMediaStream(const std::string& stream_id) { if (mediaStreamMap.count(stream_id) == 0) @@ -337,15 +352,16 @@ namespace WebRTC webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) { - nvVideoCapturerUnique = std::make_unique(); - nvVideoCapturer = nvVideoCapturerUnique.get(); - nvVideoCapturer->InitializeEncoder(width, height); - pDummyVideoEncoderFactory->SetCapturer(nvVideoCapturer); + NvVideoCapturer* pNvVideoCapturer = new NvVideoCapturer(); + pNvVideoCapturer->InitializeEncoder(width, height); + pDummyVideoEncoderFactory->AddCapturer(pNvVideoCapturer); - videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(std::move(nvVideoCapturerUnique))); - nvVideoCapturer->unityRT = frameBuffer; - nvVideoCapturer->StartEncoder(); + auto videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(pNvVideoCapturer)); + pNvVideoCapturer->unityRT = frameBuffer; + pNvVideoCapturer->StartEncoder(); + nvVideoCapturerList.push_back(pNvVideoCapturer); + mediaSteamTrackList.push_back(videoTrack); return videoTrack; } @@ -357,7 +373,8 @@ namespace WebRTC audioOptions.noise_suppression = false; audioOptions.highpass_filter = false; //TODO: label and stream id should be maintained in some way for multi-stream - audioTrack = peerConnectionFactory->CreateAudioTrack(label, peerConnectionFactory->CreateAudioSource(audioOptions)); + auto audioTrack = peerConnectionFactory->CreateAudioTrack(label, peerConnectionFactory->CreateAudioSource(audioOptions)); + mediaSteamTrackList.push_back(audioTrack); return audioTrack; } diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index 6ba184b34..27bea9879 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -49,8 +49,8 @@ namespace WebRTC PeerConnectionObject* CreatePeerConnection(int id); PeerConnectionObject* CreatePeerConnection(int id, const std::string& conf); - void EncodeFrame() { nvVideoCapturer->EncodeVideoData(); } - void StopCapturer() { nvVideoCapturer->Stop(); } + void EncodeFrame(); + void StopCapturer(); void ProcessAudioData(const float* data, int32 size) { audioDevice->ProcessAudioData(data, size); } void DeleteClient(int id) { clients.erase(id); } private: @@ -61,13 +61,10 @@ namespace WebRTC rtc::scoped_refptr peerConnectionFactory; DummyVideoEncoderFactory* pDummyVideoEncoderFactory; std::map> mediaStreamMap; + std::list> mediaSteamTrackList; - NvVideoCapturer* nvVideoCapturer; - std::unique_ptr nvVideoCapturerUnique; + std::list nvVideoCapturerList; rtc::scoped_refptr audioDevice; - - rtc::scoped_refptr videoTrack; - rtc::scoped_refptr audioTrack; }; class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp index e20840a76..61b9a8f61 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp @@ -92,8 +92,15 @@ namespace WebRTC const webrtc::SdpVideoFormat& format) { auto dummyVideoEncoder = std::make_unique(); - dummyVideoEncoder->SetKeyFrame.connect(capturer, &NvVideoCapturer::SetKeyFrame); - dummyVideoEncoder->SetRate.connect(capturer, &NvVideoCapturer::SetRate); + + { + //todo: According to condition of format choose different capturer. + NvVideoCapturer* pCapturer = *capturers.begin(); + + dummyVideoEncoder->SetKeyFrame.connect(pCapturer, &NvVideoCapturer::SetKeyFrame); + dummyVideoEncoder->SetRate.connect(pCapturer, &NvVideoCapturer::SetRate); + } + return dummyVideoEncoder; } } diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.h b/Plugin/WebRTCPlugin/DummyVideoEncoder.h index ad8f81601..db09394c0 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.h +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.h @@ -53,8 +53,8 @@ namespace WebRTC const webrtc::SdpVideoFormat& format) override; DummyVideoEncoderFactory(); - void SetCapturer(NvVideoCapturer* _capturer) { capturer = _capturer; } + void AddCapturer(NvVideoCapturer* _capturer) { capturers.push_back(_capturer); } private: - NvVideoCapturer* capturer; + std::list capturers; }; } From 3e6aa8d5b163e800260a2a70061a1bfa161742e7 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 15 Jul 2019 19:53:47 +0800 Subject: [PATCH 11/27] capturers share one nvEncoder. todo: 1.create multiple resolution texture for nvEncoder. 2.Abstract out a base class BaseEncoder. Derive AmdEncoder and softEncoder. --- Assets/Scripts/RenderStreaming.cs | 5 +-- .../Runtime/Srcipts/MediaStreamTrack.cs | 4 +-- .../Samples/Example/MediaStreamSample.cs | 4 +-- .../Tests/Runtime/MediaStreamTest.cs | 2 +- Plugin/WebRTCPlugin/Context.cpp | 6 +++- Plugin/WebRTCPlugin/Context.h | 1 + Plugin/WebRTCPlugin/NvEncoder.cpp | 32 +++++++++---------- Plugin/WebRTCPlugin/NvEncoder.h | 12 ++++--- Plugin/WebRTCPlugin/NvVideoCapturer.cpp | 6 ++-- Plugin/WebRTCPlugin/NvVideoCapturer.h | 16 +++++++--- 10 files changed, 51 insertions(+), 37 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index 085c8f108..2d1cc7a8c 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -67,10 +67,11 @@ public IEnumerator Start() int texCount = captureCamera.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - mediaStream.AddTrack(new VideoStreamTrack(captureCamera.GetStreamTexture(i))); + VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, captureCamera.GetStreamTexture(i)); + mediaStream.AddTrack(videoTrack); } - mediaStream.AddTrack(new AudioStreamTrack()); + mediaStream.AddTrack(new AudioStreamTrack("audioTrack")); Audio.Start(); signaling = new Signaling(urlSignaling); diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index 25a716ede..2c08b378e 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -36,14 +36,14 @@ private set { } public class VideoStreamTrack : MediaStreamTrack { - public VideoStreamTrack(RenderTexture rt) : base(WebRTC.Context.CreateVideoTrack("videoTrack", rt.GetNativeTexturePtr(), rt.width, rt.height)) + public VideoStreamTrack(string label, RenderTexture rt) : base(WebRTC.Context.CreateVideoTrack(label, rt.GetNativeTexturePtr(), rt.width, rt.height)) { } } public class AudioStreamTrack : MediaStreamTrack { - public AudioStreamTrack() : base(WebRTC.Context.CreateAudioTrack("audioTrack")) + public AudioStreamTrack(string label) : base(WebRTC.Context.CreateAudioTrack(label)) { } } diff --git a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs index dc28c0d66..ae7f6f4f9 100644 --- a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs +++ b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs @@ -215,10 +215,10 @@ void Call() int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - mediaStream.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i))); + mediaStream.AddTrack(new VideoStreamTrack("videoTrack"+1, cam.GetStreamTexture(i))); } - mediaStream.AddTrack(new AudioStreamTrack()); + mediaStream.AddTrack(new AudioStreamTrack("audioTrack")); Audio.Start(); RtImage.texture = cam.targetTexture; diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index ee986c32f..dff7acd69 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -53,7 +53,7 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - VideoStreamTrack videoStreamTrack = new VideoStreamTrack(cam.GetStreamTexture(i)); + VideoStreamTrack videoStreamTrack = new VideoStreamTrack("videoTrack"+i,cam.GetStreamTexture(i)); mediaStream.AddTrack(videoStreamTrack); pc1Senders.Add(peer1.AddTrack(videoStreamTrack)); } diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index 593d7e800..c63bdaed0 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -292,6 +292,8 @@ namespace WebRTC rtc::InitializeSSL(); audioDevice = new rtc::RefCountedObject(); + nvEncoder = new NvEncoder(); + auto dummyVideoEncoderFactory = std::make_unique(); pDummyVideoEncoderFactory = dummyVideoEncoderFactory.get(); @@ -317,6 +319,8 @@ namespace WebRTC mediaSteamTrackList.clear(); mediaStreamMap.clear(); nvVideoCapturerList.clear(); + delete nvEncoder; + nvEncoder = NULL; workerThread->Quit(); workerThread.reset(); @@ -352,7 +356,7 @@ namespace WebRTC webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) { - NvVideoCapturer* pNvVideoCapturer = new NvVideoCapturer(); + NvVideoCapturer* pNvVideoCapturer = new NvVideoCapturer(nvEncoder, width, height); pNvVideoCapturer->InitializeEncoder(width, height); pDummyVideoEncoderFactory->AddCapturer(pNvVideoCapturer); diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index 27bea9879..6ed5f3df1 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -65,6 +65,7 @@ namespace WebRTC std::list nvVideoCapturerList; rtc::scoped_refptr audioDevice; + NvEncoder* nvEncoder; }; class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver diff --git a/Plugin/WebRTCPlugin/NvEncoder.cpp b/Plugin/WebRTCPlugin/NvEncoder.cpp index c12e17d78..733ad43d1 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.cpp +++ b/Plugin/WebRTCPlugin/NvEncoder.cpp @@ -6,11 +6,11 @@ namespace WebRTC { - NvEncoder::NvEncoder(int width, int height) :width(width), height(height) + NvEncoder::NvEncoder() { - LogPrint(StringFormat("width is %d, height is %d", width, height).c_str()); + LogPrint(StringFormat("width is %d, height is %d", encodeWidth, encodeHeight).c_str()); checkf(g_D3D11Device != nullptr, "D3D11Device is invalid"); - checkf(width > 0 && height > 0, "Invalid width or height!"); + checkf(encodeWidth > 0 && encodeHeight > 0, "Invalid width or height!"); bool result = true; #pragma region open an encode session //open an encode session @@ -25,10 +25,10 @@ namespace WebRTC #pragma endregion #pragma region set initialization parameters nvEncInitializeParams.version = NV_ENC_INITIALIZE_PARAMS_VER; - nvEncInitializeParams.encodeWidth = width; - nvEncInitializeParams.encodeHeight = height; - nvEncInitializeParams.darWidth = width; - nvEncInitializeParams.darHeight = height; + nvEncInitializeParams.encodeWidth = encodeWidth; + nvEncInitializeParams.encodeHeight = encodeHeight; + nvEncInitializeParams.darWidth = encodeWidth; + nvEncInitializeParams.darHeight = encodeHeight; nvEncInitializeParams.encodeGUID = NV_ENC_CODEC_H264_GUID; nvEncInitializeParams.presetGUID = NV_ENC_PRESET_LOW_LATENCY_HQ_GUID; nvEncInitializeParams.frameRateNum = frameRate; @@ -87,7 +87,7 @@ namespace WebRTC } - void NvEncoder::UpdateSettings() + void NvEncoder::UpdateSettings(int width, int height) { bool settingChanged = false; if (nvEncConfig.rcParams.averageBitRate != bitRate) @@ -121,9 +121,9 @@ namespace WebRTC } } //entry for encoding a frame - void NvEncoder::EncodeFrame() + void NvEncoder::EncodeFrame(int width, int height) { - UpdateSettings(); + UpdateSettings(width, height); uint32 bufferIndexToWrite = frameCount % bufferedFrameNum; Frame& frame = bufferedFrames[bufferIndexToWrite]; #pragma region set frame params @@ -140,8 +140,8 @@ namespace WebRTC picParams.pictureStruct = NV_ENC_PIC_STRUCT_FRAME; picParams.inputBuffer = frame.inputFrame.mappedResource; picParams.bufferFmt = frame.inputFrame.bufferFormat; - picParams.inputWidth = nvEncInitializeParams.encodeWidth; - picParams.inputHeight = nvEncInitializeParams.encodeHeight; + picParams.inputWidth = encodeWidth; + picParams.inputHeight = encodeHeight; picParams.outputBitstream = frame.outputFrame; picParams.inputTimeStamp = frameCount; #pragma endregion @@ -191,8 +191,8 @@ namespace WebRTC { ID3D11Texture2D* inputTextures = nullptr; D3D11_TEXTURE2D_DESC desc = { 0 }; - desc.Width = width; - desc.Height = height; + desc.Width = encodeWidth; + desc.Height = encodeHeight; desc.MipLevels = 1; desc.ArraySize = 1; desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; @@ -212,8 +212,8 @@ namespace WebRTC if (!registerResource.resourceToRegister) LogPrint("resource is not initialized"); - registerResource.width = width; - registerResource.height = height; + registerResource.width = encodeWidth; + registerResource.height = encodeHeight; LogPrint(StringFormat("nvEncRegisterResource: width is %d, height is %d", registerResource.width, registerResource.height).c_str()); registerResource.bufferFormat = NV_ENC_BUFFER_FORMAT_ARGB; checkf(NV_RESULT((errorCode = ContextManager::GetInstance()->pNvEncodeAPI->nvEncRegisterResource(pEncoderInterface, ®isterResource))), diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index 490b92f8e..f453f086c 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -29,12 +29,12 @@ namespace WebRTC }; public: - NvEncoder(int width, int height); + NvEncoder(); ~NvEncoder(); void SetRate(uint32 rate); - void UpdateSettings(); - void EncodeFrame(); + void UpdateSettings(int width, int height); + void EncodeFrame(int width, int height); bool IsSupported() const { return isNvEncoderSupported; } void SetIdrFrame() { isIdrFrame = true; } uint64 GetCurrentFrameCount() { return frameCount; } @@ -58,8 +58,10 @@ namespace WebRTC void* pEncoderInterface = nullptr; bool isNvEncoderSupported = false; bool isIdrFrame = false; - int width = 1920; - int height = 1080; + //const int encodeWidth = 1920; + //const int encodeHeight = 1080; + const int encodeWidth = 1280; + const int encodeHeight = 720; //10Mbps int bitRate = 10000000; //100Mbps diff --git a/Plugin/WebRTCPlugin/NvVideoCapturer.cpp b/Plugin/WebRTCPlugin/NvVideoCapturer.cpp index 9e9a43012..d22769a84 100644 --- a/Plugin/WebRTCPlugin/NvVideoCapturer.cpp +++ b/Plugin/WebRTCPlugin/NvVideoCapturer.cpp @@ -3,7 +3,7 @@ namespace WebRTC { - NvVideoCapturer::NvVideoCapturer() + NvVideoCapturer::NvVideoCapturer(NvEncoder* pEncoder, int _width, int _height) : nvEncoder(pEncoder), width(_width), height(_height) { set_enable_video_adapter(false); SetSupportedFormats(std::vector(1, cricket::VideoFormat(width, height, cricket::VideoFormat::FpsToInterval(framerate), cricket::FOURCC_H264))); @@ -14,7 +14,7 @@ namespace WebRTC { int curFrameNum = nvEncoder->GetCurrentFrameCount() % bufferedFrameNum; context->CopyResource(renderTextures[curFrameNum], unityRT); - nvEncoder->EncodeFrame(); + nvEncoder->EncodeFrame(width, height); } } void NvVideoCapturer::CaptureFrame(std::vector& data) @@ -41,7 +41,7 @@ namespace WebRTC void NvVideoCapturer::InitializeEncoder(int32 width, int32 height) { - nvEncoder = std::make_unique(width, height); + //koseyile todo: one nvEncoder can connect multiple capturers. nvEncoder->CaptureFrame.connect(this, &NvVideoCapturer::CaptureFrame); } } diff --git a/Plugin/WebRTCPlugin/NvVideoCapturer.h b/Plugin/WebRTCPlugin/NvVideoCapturer.h index 5686f2216..bd6382a79 100644 --- a/Plugin/WebRTCPlugin/NvVideoCapturer.h +++ b/Plugin/WebRTCPlugin/NvVideoCapturer.h @@ -3,10 +3,15 @@ namespace WebRTC { + class testNvVideoCapturer : public cricket::VideoCapturer + { + + }; + class NvVideoCapturer : public cricket::VideoCapturer { public: - NvVideoCapturer(); + NvVideoCapturer(NvEncoder* pEncoder, int _width, int _height); void EncodeVideoData(); // Start the video capturer with the specified capture format. virtual cricket::CaptureState Start(const cricket::VideoFormat& Format) override @@ -17,7 +22,7 @@ namespace WebRTC virtual void Stop() override { captureStopped = true; - nvEncoder.reset(); + //nvEncoder.reset(); } // Check if the video capturer is running. virtual bool IsRunning() override @@ -46,11 +51,12 @@ namespace WebRTC fourccs->push_back(cricket::FOURCC_H264); return true; } - std::unique_ptr nvEncoder; + //std::unique_ptr nvEncoder; + NvEncoder* nvEncoder; //just fake info - const int32 width = 1280; - const int32 height = 720; + int32 width; + int32 height; const int32 framerate = 60; bool captureStarted = false; From c82dab4cfd438c1e0324953312003ba5b89af36d Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 15 Jul 2019 20:05:00 +0800 Subject: [PATCH 12/27] delete parameter label from MediaStream constructor --- Assets/Scripts/RenderStreaming.cs | 2 +- Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs | 8 ++++---- .../com.unity.webrtc/Samples/Example/MediaStreamSample.cs | 2 +- .../com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index 2d1cc7a8c..bb279ccda 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -63,7 +63,7 @@ public IEnumerator Start() } captureCamera.CreateRenderStreamTexture(1280, 720); - mediaStream = new MediaStream("MediaStream"); + mediaStream = new MediaStream(); int texCount = captureCamera.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 5d82eca07..76d899ac3 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -11,14 +11,14 @@ public class MediaStream internal IntPtr nativePtr; protected List mediaStreamTrackList = new List(); - public MediaStream(string label) : base() + public MediaStream() : base() { - nativePtr = WebRTC.Context.CreateMediaStream(label); + nativePtr = WebRTC.Context.CreateMediaStream("MediaStream"); } - public MediaStream(string label, MediaStreamTrack[] tracks) : base() + public MediaStream(MediaStreamTrack[] tracks) : base() { - nativePtr = WebRTC.Context.CreateMediaStream(label); + nativePtr = WebRTC.Context.CreateMediaStream("MediaStream"); foreach (var t in tracks) { diff --git a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs index ae7f6f4f9..d42856a03 100644 --- a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs +++ b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs @@ -211,7 +211,7 @@ void Call() dataChannel = pc1.CreateDataChannel("data", ref conf); cam.CreateRenderStreamTexture(1280, 720); - mediaStream = new MediaStream("mediaStream"); + mediaStream = new MediaStream(); int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index dff7acd69..b4389c529 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -49,7 +49,7 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() cam.CreateRenderStreamTexture(1280, 720); - MediaStream mediaStream = new MediaStream("mediaStream"); + MediaStream mediaStream = new MediaStream(); int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { From cdf98eafca68fd745d4430f4f277d9ead276f851 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Tue, 16 Jul 2019 13:56:19 +0800 Subject: [PATCH 13/27] delete test code. --- Plugin/WebRTCPlugin/NvVideoCapturer.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Plugin/WebRTCPlugin/NvVideoCapturer.h b/Plugin/WebRTCPlugin/NvVideoCapturer.h index bd6382a79..426e7f766 100644 --- a/Plugin/WebRTCPlugin/NvVideoCapturer.h +++ b/Plugin/WebRTCPlugin/NvVideoCapturer.h @@ -3,11 +3,6 @@ namespace WebRTC { - class testNvVideoCapturer : public cricket::VideoCapturer - { - - }; - class NvVideoCapturer : public cricket::VideoCapturer { public: From c88c3229dc7a248d59f74f9c4340843706fb52e8 Mon Sep 17 00:00:00 2001 From: Kazuki Matsumoto <1132081+karasusan@users.noreply.github.com> Date: Mon, 22 Jul 2019 08:31:03 +0900 Subject: [PATCH 14/27] changed instance type to build on Yamato --- .yamato/upm-ci-webrtc.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.yamato/upm-ci-webrtc.yml b/.yamato/upm-ci-webrtc.yml index 987be2ceb..82b1b0b01 100644 --- a/.yamato/upm-ci-webrtc.yml +++ b/.yamato/upm-ci-webrtc.yml @@ -4,7 +4,7 @@ editors: - version: trunk platforms: - name: win - type: Unity::VM + type: Unity::VM::GPU # currently the projects depends MSBuild. we should replace CMake # image: package-ci/win10:stable image: renderstreaming/win10:latest @@ -101,4 +101,4 @@ publish: - .yamato/upm-ci-webrtc.yml#test_{{ platform.name }}_{{ editor.version }} {% endfor %} {% endfor %} -{% endfor %} \ No newline at end of file +{% endfor %} From 0a13fce55182fb415ff02ac4b66d23d6b18e363c Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 22 Jul 2019 11:16:59 +0800 Subject: [PATCH 15/27] remove RTCPeerConnection when the RTCPeerConnection is disconnected. --- Assets/Scripts/RenderStreaming.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index bb279ccda..b2cad26fb 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -148,7 +148,8 @@ IEnumerator GetOffer() { if(state == RTCIceConnectionState.Disconnected) { - pc.Close(); + pc.Close(); + pcs.Remove(offer.connectionId); } }); //make video bit rate starts at 16000kbits, and 160000kbits at max. @@ -216,6 +217,7 @@ IEnumerator GetCandidate() { continue; } + foreach (var candidate in candidateContainer.candidates) { RTCIceCandidate _candidate = default; From 52138b13347b8b26f3765b1b921be862d6159fdd Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 22 Jul 2019 12:44:10 +0800 Subject: [PATCH 16/27] add auto testing to check multiple video tracks. --- Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs | 4 ++-- Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 76d899ac3..47c3a27ff 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -82,7 +82,7 @@ public static RenderTexture GetStreamTexture(this Camera cam, int index) { return webRTCTextures[index]; } - public static void CreateRenderStreamTexture(this Camera cam, int width, int height) + public static void CreateRenderStreamTexture(this Camera cam, int width, int height, int count = 1) { if (camCopyRts.Count > 0) { @@ -92,7 +92,7 @@ public static void CreateRenderStreamTexture(this Camera cam, int width, int hei camRenderTexture = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32); camRenderTexture.Create(); - int mipCount = 1; + int mipCount = count; for (int i = 1, mipLevel = 1; i <= mipCount; ++i, mipLevel *= 2) { RenderTexture webRtcTex = new RenderTexture(width / mipLevel, height / mipLevel, 0, RenderTextureFormat.BGRA32); diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index b4389c529..9b0e2081c 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -48,7 +48,7 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() }); - cam.CreateRenderStreamTexture(1280, 720); + cam.CreateRenderStreamTexture(1280, 720, 2); MediaStream mediaStream = new MediaStream(); int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) From 09ab52248b2d405c5e98e8d4dd8ed5bc02437fda Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 22 Jul 2019 12:59:13 +0800 Subject: [PATCH 17/27] rename NvVideoCapturer to UnityVideoCapturer --- Plugin/WebRTCPlugin/Context.cpp | 19 +++++---- Plugin/WebRTCPlugin/Context.h | 4 +- Plugin/WebRTCPlugin/DummyVideoEncoder.cpp | 8 ++-- Plugin/WebRTCPlugin/DummyVideoEncoder.h | 6 +-- Plugin/WebRTCPlugin/NvEncoder.cpp | 41 +++++++++++-------- Plugin/WebRTCPlugin/NvEncoder.h | 8 ++-- ...deoCapturer.cpp => UnityVideoCapturer.cpp} | 18 ++++---- ...NvVideoCapturer.h => UnityVideoCapturer.h} | 4 +- Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj | 4 +- .../WebRTCPlugin/WebRTCPlugin.vcxproj.filters | 12 +++--- 10 files changed, 67 insertions(+), 57 deletions(-) rename Plugin/WebRTCPlugin/{NvVideoCapturer.cpp => UnityVideoCapturer.cpp} (66%) rename Plugin/WebRTCPlugin/{NvVideoCapturer.h => UnityVideoCapturer.h} (95%) diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index c63bdaed0..ac6c009b6 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -330,7 +330,7 @@ namespace WebRTC void Context::EncodeFrame() { - for (std::list::iterator it= nvVideoCapturerList.begin(); it!= nvVideoCapturerList.end(); ++it) + for (std::list::iterator it= nvVideoCapturerList.begin(); it!= nvVideoCapturerList.end(); ++it) { (*it)->EncodeVideoData(); } @@ -338,7 +338,7 @@ namespace WebRTC void Context::StopCapturer() { - for (std::list::iterator it = nvVideoCapturerList.begin(); it != nvVideoCapturerList.end(); ++it) + for (std::list::iterator it = nvVideoCapturerList.begin(); it != nvVideoCapturerList.end(); ++it) { (*it)->Stop(); } @@ -356,15 +356,16 @@ namespace WebRTC webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) { - NvVideoCapturer* pNvVideoCapturer = new NvVideoCapturer(nvEncoder, width, height); - pNvVideoCapturer->InitializeEncoder(width, height); - pDummyVideoEncoderFactory->AddCapturer(pNvVideoCapturer); + nvEncoder->InitEncoder(width, height); + UnityVideoCapturer* pUnityVideoCapturer = new UnityVideoCapturer(nvEncoder, width, height); + pUnityVideoCapturer->InitializeEncoder(width, height); + pDummyVideoEncoderFactory->AddCapturer(pUnityVideoCapturer); - auto videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(pNvVideoCapturer)); - pNvVideoCapturer->unityRT = frameBuffer; - pNvVideoCapturer->StartEncoder(); + auto videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(pUnityVideoCapturer)); + pUnityVideoCapturer->unityRT = frameBuffer; + pUnityVideoCapturer->StartEncoder(); - nvVideoCapturerList.push_back(pNvVideoCapturer); + nvVideoCapturerList.push_back(pUnityVideoCapturer); mediaSteamTrackList.push_back(videoTrack); return videoTrack; } diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index 6ed5f3df1..f577d0a4a 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -2,7 +2,7 @@ #include "DummyAudioDevice.h" #include "DummyVideoEncoder.h" #include "PeerConnectionObject.h" -#include "NvVideoCapturer.h" +#include "UnityVideoCapturer.h" namespace WebRTC @@ -63,7 +63,7 @@ namespace WebRTC std::map> mediaStreamMap; std::list> mediaSteamTrackList; - std::list nvVideoCapturerList; + std::list nvVideoCapturerList; rtc::scoped_refptr audioDevice; NvEncoder* nvEncoder; }; diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp index 61b9a8f61..09500f4f2 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp @@ -1,6 +1,6 @@ #include "pch.h" #include "DummyVideoEncoder.h" -#include "NvVideoCapturer.h" +#include "UnityVideoCapturer.h" #include namespace WebRTC @@ -95,10 +95,10 @@ namespace WebRTC { //todo: According to condition of format choose different capturer. - NvVideoCapturer* pCapturer = *capturers.begin(); + UnityVideoCapturer* pCapturer = *capturers.begin(); - dummyVideoEncoder->SetKeyFrame.connect(pCapturer, &NvVideoCapturer::SetKeyFrame); - dummyVideoEncoder->SetRate.connect(pCapturer, &NvVideoCapturer::SetRate); + dummyVideoEncoder->SetKeyFrame.connect(pCapturer, &UnityVideoCapturer::SetKeyFrame); + dummyVideoEncoder->SetRate.connect(pCapturer, &UnityVideoCapturer::SetRate); } return dummyVideoEncoder; diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.h b/Plugin/WebRTCPlugin/DummyVideoEncoder.h index db09394c0..88b467c0b 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.h +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.h @@ -2,7 +2,7 @@ namespace WebRTC { - class NvVideoCapturer; + class UnityVideoCapturer; class DummyVideoEncoder : public webrtc::VideoEncoder { public: @@ -53,8 +53,8 @@ namespace WebRTC const webrtc::SdpVideoFormat& format) override; DummyVideoEncoderFactory(); - void AddCapturer(NvVideoCapturer* _capturer) { capturers.push_back(_capturer); } + void AddCapturer(UnityVideoCapturer* _capturer) { capturers.push_back(_capturer); } private: - std::list capturers; + std::list capturers; }; } diff --git a/Plugin/WebRTCPlugin/NvEncoder.cpp b/Plugin/WebRTCPlugin/NvEncoder.cpp index 733ad43d1..59e1678c2 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.cpp +++ b/Plugin/WebRTCPlugin/NvEncoder.cpp @@ -22,7 +22,26 @@ namespace WebRTC result = NV_RESULT((errorCode = ContextManager::GetInstance()->pNvEncodeAPI->nvEncOpenEncodeSessionEx(&openEncdoeSessionExParams, &pEncoderInterface))); checkf(result, "Unable to open NvEnc encode session"); LogPrint(StringFormat("OpenEncodeSession Error is %d", errorCode).c_str()); -#pragma endregion +#pragma endregion + } + + NvEncoder::~NvEncoder() + { + ReleaseEncoderResources(); + if (pEncoderInterface) + { + bool result = NV_RESULT(ContextManager::GetInstance()->pNvEncodeAPI->nvEncDestroyEncoder(pEncoderInterface)); + checkf(result, "Failed to destroy NV encoder interface"); + pEncoderInterface = nullptr; + } + + } + + void NvEncoder::InitEncoder(int width, int height) + { + encodeWidth = width; + encodeHeight = height; + bool result = true; #pragma region set initialization parameters nvEncInitializeParams.version = NV_ENC_INITIALIZE_PARAMS_VER; nvEncInitializeParams.encodeWidth = encodeWidth; @@ -37,8 +56,8 @@ namespace WebRTC nvEncInitializeParams.reportSliceOffsets = 0; nvEncInitializeParams.enableSubFrameWrite = 0; nvEncInitializeParams.encodeConfig = &nvEncConfig; - nvEncInitializeParams.maxEncodeWidth = 3840; - nvEncInitializeParams.maxEncodeHeight = 2160; + nvEncInitializeParams.maxEncodeWidth = encodeWidth;//3840; + nvEncInitializeParams.maxEncodeHeight = encodeHeight;//2160; #pragma endregion #pragma region get preset ocnfig and set it NV_ENC_PRESET_CONFIG presetConfig = { 0 }; @@ -74,17 +93,7 @@ namespace WebRTC #pragma endregion InitEncoderResources(); isNvEncoderSupported = true; - } - NvEncoder::~NvEncoder() - { - ReleaseEncoderResources(); - if (pEncoderInterface) - { - bool result = NV_RESULT(ContextManager::GetInstance()->pNvEncodeAPI->nvEncDestroyEncoder(pEncoderInterface)); - checkf(result, "Failed to destroy NV encoder interface"); - pEncoderInterface = nullptr; - } - + isInitialize = true; } void NvEncoder::UpdateSettings(int width, int height) @@ -140,8 +149,8 @@ namespace WebRTC picParams.pictureStruct = NV_ENC_PIC_STRUCT_FRAME; picParams.inputBuffer = frame.inputFrame.mappedResource; picParams.bufferFmt = frame.inputFrame.bufferFormat; - picParams.inputWidth = encodeWidth; - picParams.inputHeight = encodeHeight; + picParams.inputWidth = width; + picParams.inputHeight = height; picParams.outputBitstream = frame.outputFrame; picParams.inputTimeStamp = frameCount; #pragma endregion diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index f453f086c..5f34de852 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -39,6 +39,7 @@ namespace WebRTC void SetIdrFrame() { isIdrFrame = true; } uint64 GetCurrentFrameCount() { return frameCount; } sigslot::signal1&> CaptureFrame; + void InitEncoder(int width, int height); void InitEncoderResources(); private: @@ -57,11 +58,10 @@ namespace WebRTC uint64 frameCount = 0; void* pEncoderInterface = nullptr; bool isNvEncoderSupported = false; + bool isInitialize = false; bool isIdrFrame = false; - //const int encodeWidth = 1920; - //const int encodeHeight = 1080; - const int encodeWidth = 1280; - const int encodeHeight = 720; + int encodeWidth; + int encodeHeight; //10Mbps int bitRate = 10000000; //100Mbps diff --git a/Plugin/WebRTCPlugin/NvVideoCapturer.cpp b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp similarity index 66% rename from Plugin/WebRTCPlugin/NvVideoCapturer.cpp rename to Plugin/WebRTCPlugin/UnityVideoCapturer.cpp index d22769a84..b5a5c488a 100644 --- a/Plugin/WebRTCPlugin/NvVideoCapturer.cpp +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp @@ -1,14 +1,14 @@ #include "pch.h" -#include "NvVideoCapturer.h" +#include "UnityVideoCapturer.h" namespace WebRTC { - NvVideoCapturer::NvVideoCapturer(NvEncoder* pEncoder, int _width, int _height) : nvEncoder(pEncoder), width(_width), height(_height) + UnityVideoCapturer::UnityVideoCapturer(NvEncoder* pEncoder, int _width, int _height) : nvEncoder(pEncoder), width(_width), height(_height) { set_enable_video_adapter(false); SetSupportedFormats(std::vector(1, cricket::VideoFormat(width, height, cricket::VideoFormat::FpsToInterval(framerate), cricket::FOURCC_H264))); } - void NvVideoCapturer::EncodeVideoData() + void UnityVideoCapturer::EncodeVideoData() { if (captureStarted && !captureStopped) { @@ -17,7 +17,7 @@ namespace WebRTC nvEncoder->EncodeFrame(width, height); } } - void NvVideoCapturer::CaptureFrame(std::vector& data) + void UnityVideoCapturer::CaptureFrame(std::vector& data) { rtc::scoped_refptr buffer = new rtc::RefCountedObject(width, height, data); int64 timestamp = rtc::TimeMillis(); @@ -25,23 +25,23 @@ namespace WebRTC videoFrame.set_ntp_time_ms(timestamp); OnFrame(videoFrame, width, height); } - void NvVideoCapturer::StartEncoder() + void UnityVideoCapturer::StartEncoder() { captureStarted = true; SetKeyFrame(); } - void NvVideoCapturer::SetKeyFrame() + void UnityVideoCapturer::SetKeyFrame() { nvEncoder->SetIdrFrame(); } - void NvVideoCapturer::SetRate(uint32 rate) + void UnityVideoCapturer::SetRate(uint32 rate) { nvEncoder->SetRate(rate); } - void NvVideoCapturer::InitializeEncoder(int32 width, int32 height) + void UnityVideoCapturer::InitializeEncoder(int32 width, int32 height) { //koseyile todo: one nvEncoder can connect multiple capturers. - nvEncoder->CaptureFrame.connect(this, &NvVideoCapturer::CaptureFrame); + nvEncoder->CaptureFrame.connect(this, &UnityVideoCapturer::CaptureFrame); } } diff --git a/Plugin/WebRTCPlugin/NvVideoCapturer.h b/Plugin/WebRTCPlugin/UnityVideoCapturer.h similarity index 95% rename from Plugin/WebRTCPlugin/NvVideoCapturer.h rename to Plugin/WebRTCPlugin/UnityVideoCapturer.h index 426e7f766..5b0ac8881 100644 --- a/Plugin/WebRTCPlugin/NvVideoCapturer.h +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.h @@ -3,10 +3,10 @@ namespace WebRTC { - class NvVideoCapturer : public cricket::VideoCapturer + class UnityVideoCapturer : public cricket::VideoCapturer { public: - NvVideoCapturer(NvEncoder* pEncoder, int _width, int _height); + UnityVideoCapturer(NvEncoder* pEncoder, int _width, int _height); void EncodeVideoData(); // Start the video capturer with the specified capture format. virtual cricket::CaptureState Start(const cricket::VideoFormat& Format) override diff --git a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj index 54636a0a8..e962f41fb 100644 --- a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj +++ b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj @@ -177,7 +177,7 @@ - + @@ -190,7 +190,7 @@ - + Create Create diff --git a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters index 0a0d2a3a4..56f90d06b 100644 --- a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters +++ b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters @@ -42,9 +42,6 @@ Header Files - - Header Files - Header Files @@ -60,6 +57,9 @@ Header Files\Unity + + Header Files + @@ -83,9 +83,6 @@ Source Files - - Source Files - Source Files @@ -95,5 +92,8 @@ Source Files + + Source Files + \ No newline at end of file From 5d00ed0babab4574ef937e039704e12eb3563218 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 22 Jul 2019 15:03:08 +0800 Subject: [PATCH 18/27] add abstract bass class : UnityEncoder. --- Plugin/WebRTCPlugin/Context.cpp | 10 ++++---- Plugin/WebRTCPlugin/Context.h | 4 ++-- Plugin/WebRTCPlugin/NvEncoder.h | 5 ++-- Plugin/WebRTCPlugin/UnityEncoder.cpp | 13 +++++++++++ Plugin/WebRTCPlugin/UnityEncoder.h | 23 +++++++++++++++++++ Plugin/WebRTCPlugin/UnityVideoCapturer.cpp | 2 +- Plugin/WebRTCPlugin/UnityVideoCapturer.h | 6 ++--- Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj | 2 ++ .../WebRTCPlugin/WebRTCPlugin.vcxproj.filters | 6 +++++ 9 files changed, 57 insertions(+), 14 deletions(-) create mode 100644 Plugin/WebRTCPlugin/UnityEncoder.cpp create mode 100644 Plugin/WebRTCPlugin/UnityEncoder.h diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index ac6c009b6..380fd8d1e 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -292,7 +292,7 @@ namespace WebRTC rtc::InitializeSSL(); audioDevice = new rtc::RefCountedObject(); - nvEncoder = new NvEncoder(); + pUnityEncoder = new NvEncoder(); auto dummyVideoEncoderFactory = std::make_unique(); pDummyVideoEncoderFactory = dummyVideoEncoderFactory.get(); @@ -319,8 +319,8 @@ namespace WebRTC mediaSteamTrackList.clear(); mediaStreamMap.clear(); nvVideoCapturerList.clear(); - delete nvEncoder; - nvEncoder = NULL; + delete pUnityEncoder; + pUnityEncoder = NULL; workerThread->Quit(); workerThread.reset(); @@ -356,8 +356,8 @@ namespace WebRTC webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) { - nvEncoder->InitEncoder(width, height); - UnityVideoCapturer* pUnityVideoCapturer = new UnityVideoCapturer(nvEncoder, width, height); + pUnityEncoder->InitEncoder(width, height); + UnityVideoCapturer* pUnityVideoCapturer = new UnityVideoCapturer(pUnityEncoder, width, height); pUnityVideoCapturer->InitializeEncoder(width, height); pDummyVideoEncoderFactory->AddCapturer(pUnityVideoCapturer); diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index f577d0a4a..855795bc2 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -3,7 +3,7 @@ #include "DummyVideoEncoder.h" #include "PeerConnectionObject.h" #include "UnityVideoCapturer.h" - +#include "NvEncoder.h" namespace WebRTC { @@ -65,7 +65,7 @@ namespace WebRTC std::list nvVideoCapturerList; rtc::scoped_refptr audioDevice; - NvEncoder* nvEncoder; + UnityEncoder* pUnityEncoder; }; class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index 5f34de852..3468e0531 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -4,11 +4,12 @@ #include "nvEncodeAPI.h" #include #include +#include "UnityEncoder.h" namespace WebRTC { using OutputFrame = NV_ENC_OUTPUT_PTR; - class NvEncoder + class NvEncoder : public UnityEncoder { private: struct InputFrame @@ -38,12 +39,10 @@ namespace WebRTC bool IsSupported() const { return isNvEncoderSupported; } void SetIdrFrame() { isIdrFrame = true; } uint64 GetCurrentFrameCount() { return frameCount; } - sigslot::signal1&> CaptureFrame; void InitEncoder(int width, int height); void InitEncoderResources(); private: - void LoadNvEncApi(); void ReleaseFrameInputBuffer(Frame& frame); void ReleaseEncoderResources(); void ProcessEncodedFrame(Frame& frame); diff --git a/Plugin/WebRTCPlugin/UnityEncoder.cpp b/Plugin/WebRTCPlugin/UnityEncoder.cpp new file mode 100644 index 000000000..52c58150e --- /dev/null +++ b/Plugin/WebRTCPlugin/UnityEncoder.cpp @@ -0,0 +1,13 @@ +#include "pch.h" +#include "UnityEncoder.h" + +namespace WebRTC +{ + UnityEncoder::UnityEncoder() + { + } + + UnityEncoder::~UnityEncoder() + { + } +} diff --git a/Plugin/WebRTCPlugin/UnityEncoder.h b/Plugin/WebRTCPlugin/UnityEncoder.h new file mode 100644 index 000000000..f9a622c24 --- /dev/null +++ b/Plugin/WebRTCPlugin/UnityEncoder.h @@ -0,0 +1,23 @@ +#pragma once + +namespace WebRTC +{ + + class UnityEncoder + { + public: + UnityEncoder(); + virtual ~UnityEncoder(); + + sigslot::signal1&> CaptureFrame; + virtual void SetRate(uint32 rate) = 0; + virtual void UpdateSettings(int width, int height) = 0; + virtual void EncodeFrame(int width, int height) = 0; + virtual bool IsSupported() const = 0; + virtual void SetIdrFrame() = 0; + virtual uint64 GetCurrentFrameCount() = 0; + virtual void InitEncoder(int width, int height) = 0; + virtual void InitEncoderResources() = 0; + }; +} + diff --git a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp index b5a5c488a..e3baf8c29 100644 --- a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp @@ -3,7 +3,7 @@ namespace WebRTC { - UnityVideoCapturer::UnityVideoCapturer(NvEncoder* pEncoder, int _width, int _height) : nvEncoder(pEncoder), width(_width), height(_height) + UnityVideoCapturer::UnityVideoCapturer(UnityEncoder* pEncoder, int _width, int _height) : nvEncoder(pEncoder), width(_width), height(_height) { set_enable_video_adapter(false); SetSupportedFormats(std::vector(1, cricket::VideoFormat(width, height, cricket::VideoFormat::FpsToInterval(framerate), cricket::FOURCC_H264))); diff --git a/Plugin/WebRTCPlugin/UnityVideoCapturer.h b/Plugin/WebRTCPlugin/UnityVideoCapturer.h index 5b0ac8881..2c6577bdf 100644 --- a/Plugin/WebRTCPlugin/UnityVideoCapturer.h +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.h @@ -1,12 +1,12 @@ #pragma once -#include "NvEncoder.h" +#include "UnityEncoder.h" namespace WebRTC { class UnityVideoCapturer : public cricket::VideoCapturer { public: - UnityVideoCapturer(NvEncoder* pEncoder, int _width, int _height); + UnityVideoCapturer(UnityEncoder* pEncoder, int _width, int _height); void EncodeVideoData(); // Start the video capturer with the specified capture format. virtual cricket::CaptureState Start(const cricket::VideoFormat& Format) override @@ -47,7 +47,7 @@ namespace WebRTC return true; } //std::unique_ptr nvEncoder; - NvEncoder* nvEncoder; + UnityEncoder* nvEncoder; //just fake info int32 width; diff --git a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj index e962f41fb..e295f9077 100644 --- a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj +++ b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj @@ -177,6 +177,7 @@ + @@ -190,6 +191,7 @@ + Create diff --git a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters index 56f90d06b..d3885ee91 100644 --- a/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters +++ b/Plugin/WebRTCPlugin/WebRTCPlugin.vcxproj.filters @@ -60,6 +60,9 @@ Header Files + + Header Files + @@ -95,5 +98,8 @@ Source Files + + Source Files + \ No newline at end of file From ed8916a9ef7f3ab61772c0bf6479a1f538c64500 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 22 Jul 2019 17:06:14 +0800 Subject: [PATCH 19/27] let DummyVideoEncoderFactory manage the encoder of platform. --- Plugin/WebRTCPlugin/Context.cpp | 11 +++---- Plugin/WebRTCPlugin/Context.h | 2 +- Plugin/WebRTCPlugin/DummyVideoEncoder.cpp | 32 ++++++++++++++++++++ Plugin/WebRTCPlugin/DummyVideoEncoder.h | 10 +++++++ Plugin/WebRTCPlugin/NvEncoder.cpp | 35 +++++++++++++--------- Plugin/WebRTCPlugin/NvEncoder.h | 2 +- Plugin/WebRTCPlugin/UnityVideoCapturer.cpp | 2 +- Plugin/WebRTCPlugin/UnityVideoCapturer.h | 2 +- 8 files changed, 73 insertions(+), 23 deletions(-) diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index 380fd8d1e..11cdc906e 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -1,5 +1,6 @@ #include "pch.h" #include "WebRTCPlugin.h" +#include "UnityEncoder.h" #include "Context.h" namespace WebRTC @@ -292,7 +293,6 @@ namespace WebRTC rtc::InitializeSSL(); audioDevice = new rtc::RefCountedObject(); - pUnityEncoder = new NvEncoder(); auto dummyVideoEncoderFactory = std::make_unique(); pDummyVideoEncoderFactory = dummyVideoEncoderFactory.get(); @@ -319,13 +319,14 @@ namespace WebRTC mediaSteamTrackList.clear(); mediaStreamMap.clear(); nvVideoCapturerList.clear(); - delete pUnityEncoder; - pUnityEncoder = NULL; workerThread->Quit(); workerThread.reset(); signalingThread->Quit(); signalingThread.reset(); + + pDummyVideoEncoderFactory->Destroy(); + } void Context::EncodeFrame() @@ -356,9 +357,9 @@ namespace WebRTC webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) { - pUnityEncoder->InitEncoder(width, height); + UnityEncoder* pUnityEncoder = pDummyVideoEncoderFactory->CreatePlatformEncoder(WebRTC::Nvidia, width, height); UnityVideoCapturer* pUnityVideoCapturer = new UnityVideoCapturer(pUnityEncoder, width, height); - pUnityVideoCapturer->InitializeEncoder(width, height); + pUnityVideoCapturer->InitializeEncoder(); pDummyVideoEncoderFactory->AddCapturer(pUnityVideoCapturer); auto videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(pUnityVideoCapturer)); diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index 855795bc2..aa4ba44fa 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -1,4 +1,5 @@ #pragma once +#include "UnityEncoder.h" #include "DummyAudioDevice.h" #include "DummyVideoEncoder.h" #include "PeerConnectionObject.h" @@ -65,7 +66,6 @@ namespace WebRTC std::list nvVideoCapturerList; rtc::scoped_refptr audioDevice; - UnityEncoder* pUnityEncoder; }; class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp index 09500f4f2..618a7435a 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp @@ -1,7 +1,9 @@ #include "pch.h" +#include "UnityEncoder.h" #include "DummyVideoEncoder.h" #include "UnityVideoCapturer.h" #include +#include "NvEncoder.h" namespace WebRTC { @@ -73,6 +75,15 @@ namespace WebRTC } + void DummyVideoEncoderFactory::Destroy() + { + for (std::list::iterator it = unityEncoders.begin(); it!= unityEncoders.end(); ++it) + { + delete *it; + } + unityEncoders.clear(); + } + std::vector DummyVideoEncoderFactory::GetSupportedFormats() const { const absl::optional profileLevelId = @@ -103,4 +114,25 @@ namespace WebRTC return dummyVideoEncoder; } + + UnityEncoder* DummyVideoEncoderFactory::CreatePlatformEncoder(EncoderPlatform platform, int width, int height) + { + UnityEncoder* pEncoder = NULL; + switch (platform) + { + case WebRTC::Nvidia: + pEncoder = new NvEncoder(); + break; + case WebRTC::Amd: + break; + case WebRTC::Soft: + break; + default: + break; + } + pEncoder->InitEncoder(width, height); + unityEncoders.push_back(pEncoder); + return pEncoder; + } + } diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.h b/Plugin/WebRTCPlugin/DummyVideoEncoder.h index 88b467c0b..c791fbe2b 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.h +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.h @@ -38,6 +38,13 @@ namespace WebRTC webrtc::VideoBitrateAllocation lastBitrate; }; + enum EncoderPlatform + { + Nvidia, + Amd, + Soft, + }; + class DummyVideoEncoderFactory : public webrtc::VideoEncoderFactory { public: @@ -52,9 +59,12 @@ namespace WebRTC virtual std::unique_ptr CreateVideoEncoder( const webrtc::SdpVideoFormat& format) override; DummyVideoEncoderFactory(); + void Destroy(); void AddCapturer(UnityVideoCapturer* _capturer) { capturers.push_back(_capturer); } + UnityEncoder* CreatePlatformEncoder(EncoderPlatform platform, int width, int height); private: std::list capturers; + std::list unityEncoders; }; } diff --git a/Plugin/WebRTCPlugin/NvEncoder.cpp b/Plugin/WebRTCPlugin/NvEncoder.cpp index 59e1678c2..fd384cd38 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.cpp +++ b/Plugin/WebRTCPlugin/NvEncoder.cpp @@ -6,23 +6,25 @@ namespace WebRTC { + void* NvEncoder::pEncoderInterface = nullptr; NvEncoder::NvEncoder() { - LogPrint(StringFormat("width is %d, height is %d", encodeWidth, encodeHeight).c_str()); - checkf(g_D3D11Device != nullptr, "D3D11Device is invalid"); - checkf(encodeWidth > 0 && encodeHeight > 0, "Invalid width or height!"); - bool result = true; + if (pEncoderInterface==nullptr) + { + bool result = true; #pragma region open an encode session - //open an encode session - NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS openEncdoeSessionExParams = { 0 }; - openEncdoeSessionExParams.version = NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER; - openEncdoeSessionExParams.device = g_D3D11Device; - openEncdoeSessionExParams.deviceType = NV_ENC_DEVICE_TYPE_DIRECTX; - openEncdoeSessionExParams.apiVersion = NVENCAPI_VERSION; - result = NV_RESULT((errorCode = ContextManager::GetInstance()->pNvEncodeAPI->nvEncOpenEncodeSessionEx(&openEncdoeSessionExParams, &pEncoderInterface))); - checkf(result, "Unable to open NvEnc encode session"); - LogPrint(StringFormat("OpenEncodeSession Error is %d", errorCode).c_str()); -#pragma endregion + //open an encode session + NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS openEncdoeSessionExParams = { 0 }; + openEncdoeSessionExParams.version = NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER; + openEncdoeSessionExParams.device = g_D3D11Device; + openEncdoeSessionExParams.deviceType = NV_ENC_DEVICE_TYPE_DIRECTX; + openEncdoeSessionExParams.apiVersion = NVENCAPI_VERSION; + result = NV_RESULT((errorCode = ContextManager::GetInstance()->pNvEncodeAPI->nvEncOpenEncodeSessionEx(&openEncdoeSessionExParams, &pEncoderInterface))); + checkf(result, "Unable to open NvEnc encode session"); + LogPrint(StringFormat("OpenEncodeSession Error is %d", errorCode).c_str()); +#pragma endregion + } + } NvEncoder::~NvEncoder() @@ -41,6 +43,11 @@ namespace WebRTC { encodeWidth = width; encodeHeight = height; + + LogPrint(StringFormat("width is %d, height is %d", encodeWidth, encodeHeight).c_str()); + checkf(g_D3D11Device != nullptr, "D3D11Device is invalid"); + checkf(encodeWidth > 0 && encodeHeight > 0, "Invalid width or height!"); + bool result = true; #pragma region set initialization parameters nvEncInitializeParams.version = NV_ENC_INITIALIZE_PARAMS_VER; diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index 3468e0531..93b097538 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -55,7 +55,7 @@ namespace WebRTC _NVENCSTATUS errorCode; Frame bufferedFrames[bufferedFrameNum]; uint64 frameCount = 0; - void* pEncoderInterface = nullptr; + static void* pEncoderInterface; bool isNvEncoderSupported = false; bool isInitialize = false; bool isIdrFrame = false; diff --git a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp index e3baf8c29..58132fca7 100644 --- a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp @@ -39,7 +39,7 @@ namespace WebRTC nvEncoder->SetRate(rate); } - void UnityVideoCapturer::InitializeEncoder(int32 width, int32 height) + void UnityVideoCapturer::InitializeEncoder() { //koseyile todo: one nvEncoder can connect multiple capturers. nvEncoder->CaptureFrame.connect(this, &UnityVideoCapturer::CaptureFrame); diff --git a/Plugin/WebRTCPlugin/UnityVideoCapturer.h b/Plugin/WebRTCPlugin/UnityVideoCapturer.h index 2c6577bdf..94c333631 100644 --- a/Plugin/WebRTCPlugin/UnityVideoCapturer.h +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.h @@ -31,7 +31,7 @@ namespace WebRTC return false; } void StartEncoder(); - void InitializeEncoder(int32 width, int32 height); + void InitializeEncoder(); void SetKeyFrame(); void SetRate(uint32 rate); void CaptureFrame(std::vector& data); From ec8bbb1d7ade7139ab843aa3f86f318d91a49693 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 22 Jul 2019 20:06:27 +0800 Subject: [PATCH 20/27] each nvEncoder has a texture buffer. --- Plugin/WebRTCPlugin/Callback.cpp | 10 ---------- Plugin/WebRTCPlugin/NvEncoder.cpp | 21 +++++++++++---------- Plugin/WebRTCPlugin/NvEncoder.h | 3 +++ Plugin/WebRTCPlugin/UnityEncoder.h | 1 + Plugin/WebRTCPlugin/UnityVideoCapturer.cpp | 3 +-- Plugin/WebRTCPlugin/pch.h | 3 +-- 6 files changed, 17 insertions(+), 24 deletions(-) diff --git a/Plugin/WebRTCPlugin/Callback.cpp b/Plugin/WebRTCPlugin/Callback.cpp index 5682d1eaf..c22b13745 100644 --- a/Plugin/WebRTCPlugin/Callback.cpp +++ b/Plugin/WebRTCPlugin/Callback.cpp @@ -12,8 +12,6 @@ namespace WebRTC ID3D11DeviceContext* context; //d3d11 device ID3D11Device* g_D3D11Device = nullptr; - //natively created ID3D11Texture2D ptrs - UnityFrameBuffer* renderTextures[bufferedFrameNum]; } using namespace WebRTC; //get d3d11 device @@ -33,14 +31,6 @@ static void UNITY_INTERFACE_API OnGraphicsDeviceEvent(UnityGfxDeviceEventType ev } case kUnityGfxDeviceEventShutdown: { - for (auto rt : renderTextures) - { - if (rt) - { - rt->Release(); - rt = nullptr; - } - } //UnityPluginUnload not called normally s_Graphics->UnregisterDeviceEventCallback(OnGraphicsDeviceEvent); break; diff --git a/Plugin/WebRTCPlugin/NvEncoder.cpp b/Plugin/WebRTCPlugin/NvEncoder.cpp index fd384cd38..52c031e75 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.cpp +++ b/Plugin/WebRTCPlugin/NvEncoder.cpp @@ -216,7 +216,7 @@ namespace WebRTC desc.Usage = D3D11_USAGE_DEFAULT; desc.BindFlags = D3D11_BIND_RENDER_TARGET; desc.CPUAccessFlags = 0; - g_D3D11Device->CreateTexture2D(&desc, NULL, &inputTextures); + HRESULT r = g_D3D11Device->CreateTexture2D(&desc, NULL, &inputTextures); return inputTextures; } NV_ENC_REGISTERED_PTR NvEncoder::RegisterResource(void *buffer) @@ -255,15 +255,13 @@ namespace WebRTC } void NvEncoder::InitEncoderResources() { - for (uint32 i = 0; i < bufferedFrameNum; i++) - { - renderTextures[i] = AllocateInputBuffers(); - Frame& frame = bufferedFrames[i]; - frame.inputFrame.registeredResource = RegisterResource(renderTextures[i]); - frame.inputFrame.bufferFormat = NV_ENC_BUFFER_FORMAT_ARGB; - MapResources(frame.inputFrame); - frame.outputFrame = InitializeBitstreamBuffer(); - } + nvRenderTexture = AllocateInputBuffers(); + Frame& frame = bufferedFrames[0]; + frame.inputFrame.registeredResource = RegisterResource(nvRenderTexture); + frame.inputFrame.bufferFormat = NV_ENC_BUFFER_FORMAT_ARGB; + MapResources(frame.inputFrame); + frame.outputFrame = InitializeBitstreamBuffer(); + } void NvEncoder::ReleaseFrameInputBuffer(Frame& frame) { @@ -284,6 +282,9 @@ namespace WebRTC checkf(result, "Failed to destroy output buffer bit stream"); frame.outputFrame = nullptr; } + + nvRenderTexture->Release(); + nvRenderTexture = nullptr; } } diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index 93b097538..4142d9ba3 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -41,6 +41,7 @@ namespace WebRTC uint64 GetCurrentFrameCount() { return frameCount; } void InitEncoder(int width, int height); void InitEncoderResources(); + void* getRenderTexture() { return nvRenderTexture; } private: void ReleaseFrameInputBuffer(Frame& frame); @@ -68,6 +69,8 @@ namespace WebRTC //5Mbps const int minBitRate = 5000000; int frameRate = 45; + + UnityFrameBuffer* nvRenderTexture; }; } diff --git a/Plugin/WebRTCPlugin/UnityEncoder.h b/Plugin/WebRTCPlugin/UnityEncoder.h index f9a622c24..07bd0dcc5 100644 --- a/Plugin/WebRTCPlugin/UnityEncoder.h +++ b/Plugin/WebRTCPlugin/UnityEncoder.h @@ -18,6 +18,7 @@ namespace WebRTC virtual uint64 GetCurrentFrameCount() = 0; virtual void InitEncoder(int width, int height) = 0; virtual void InitEncoderResources() = 0; + virtual void* getRenderTexture() = 0; }; } diff --git a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp index 58132fca7..c980679cd 100644 --- a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp @@ -12,8 +12,7 @@ namespace WebRTC { if (captureStarted && !captureStopped) { - int curFrameNum = nvEncoder->GetCurrentFrameCount() % bufferedFrameNum; - context->CopyResource(renderTextures[curFrameNum], unityRT); + context->CopyResource((ID3D11Resource*)nvEncoder->getRenderTexture(), unityRT); nvEncoder->EncodeFrame(width, height); } } diff --git a/Plugin/WebRTCPlugin/pch.h b/Plugin/WebRTCPlugin/pch.h index 0be103dff..231ba92ba 100644 --- a/Plugin/WebRTCPlugin/pch.h +++ b/Plugin/WebRTCPlugin/pch.h @@ -91,8 +91,7 @@ namespace WebRTC using int32 = signed int; using int64 = signed long long; - const uint32 bufferedFrameNum = 3; - extern UnityFrameBuffer* renderTextures[bufferedFrameNum]; + const uint32 bufferedFrameNum = 1; extern ID3D11DeviceContext* context; extern ID3D11Device* g_D3D11Device; } From 3e248ea5f35cfb532a27c139871a4f2dea601362 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Tue, 23 Jul 2019 11:35:19 +0800 Subject: [PATCH 21/27] delete comment code. --- Plugin/WebRTCPlugin/UnityVideoCapturer.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/Plugin/WebRTCPlugin/UnityVideoCapturer.h b/Plugin/WebRTCPlugin/UnityVideoCapturer.h index 94c333631..11777dcce 100644 --- a/Plugin/WebRTCPlugin/UnityVideoCapturer.h +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.h @@ -17,7 +17,6 @@ namespace WebRTC virtual void Stop() override { captureStopped = true; - //nvEncoder.reset(); } // Check if the video capturer is running. virtual bool IsRunning() override @@ -46,7 +45,6 @@ namespace WebRTC fourccs->push_back(cricket::FOURCC_H264); return true; } - //std::unique_ptr nvEncoder; UnityEncoder* nvEncoder; //just fake info From 309436b8afc64e617fd173f9dd7f5c87058e003f Mon Sep 17 00:00:00 2001 From: chenyuan Date: Tue, 23 Jul 2019 15:54:33 +0800 Subject: [PATCH 22/27] change pEncoderInterface to no-static. --- Plugin/WebRTCPlugin/NvEncoder.cpp | 2 +- Plugin/WebRTCPlugin/NvEncoder.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Plugin/WebRTCPlugin/NvEncoder.cpp b/Plugin/WebRTCPlugin/NvEncoder.cpp index 52c031e75..b7f737967 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.cpp +++ b/Plugin/WebRTCPlugin/NvEncoder.cpp @@ -6,7 +6,7 @@ namespace WebRTC { - void* NvEncoder::pEncoderInterface = nullptr; + NvEncoder::NvEncoder() { if (pEncoderInterface==nullptr) diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index 4142d9ba3..5761e597b 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -56,7 +56,7 @@ namespace WebRTC _NVENCSTATUS errorCode; Frame bufferedFrames[bufferedFrameNum]; uint64 frameCount = 0; - static void* pEncoderInterface; + void* pEncoderInterface = nullptr; bool isNvEncoderSupported = false; bool isInitialize = false; bool isIdrFrame = false; From ce51d0f8eaaee632f240099b69c84b7f5fe0bd85 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 5 Aug 2019 16:00:56 +0800 Subject: [PATCH 23/27] web app can switch two resolution to show. --- Assets/Scripts/RenderStreaming.cs | 42 +++++++++++++------ .../Runtime/Srcipts/MediaStream.cs | 14 +++++-- .../Runtime/Srcipts/MediaStreamTrack.cs | 2 +- .../Runtime/Srcipts/RTCPeerConnection.cs | 4 +- .../Runtime/Srcipts/WebRTC.cs | 14 +++++-- .../Samples/Example/MediaStreamSample.cs | 3 +- .../Tests/Runtime/MediaStreamTest.cs | 3 +- Plugin/WebRTCPlugin/Context.cpp | 11 ++--- Plugin/WebRTCPlugin/Context.h | 2 +- Plugin/WebRTCPlugin/DummyVideoEncoder.cpp | 10 ++--- Plugin/WebRTCPlugin/DummyVideoEncoder.h | 2 +- Plugin/WebRTCPlugin/NvEncoder.cpp | 3 +- Plugin/WebRTCPlugin/NvEncoder.h | 2 +- Plugin/WebRTCPlugin/UnityEncoder.h | 2 +- Plugin/WebRTCPlugin/WebRTCPlugin.cpp | 8 ++-- WebApp/public/scripts/app.js | 10 +++++ WebApp/public/scripts/video-player.js | 22 ++++++++++ WebApp/public/stylesheets/style.css | 17 +++++++- 18 files changed, 128 insertions(+), 43 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index b2cad26fb..eb1ceaab3 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -41,7 +41,7 @@ public class RenderStreaming : MonoBehaviour private Dictionary> mapChannels = new Dictionary>(); private RTCConfiguration conf; private string sessionId; - private MediaStream mediaStream; + private MediaStream[] mediaStreams = new MediaStream[2]; public void Awake() { @@ -62,16 +62,19 @@ public IEnumerator Start() yield break; } - captureCamera.CreateRenderStreamTexture(1280, 720); - mediaStream = new MediaStream(); + captureCamera.CreateRenderStreamTexture(1280, 720, mediaStreams.Length); + int texCount = captureCamera.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, captureCamera.GetStreamTexture(i)); - mediaStream.AddTrack(videoTrack); + int index = i; + mediaStreams[i] = new MediaStream(); + RenderTexture rt = captureCamera.GetStreamTexture(index); + VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, rt); + mediaStreams[i].AddTrack(videoTrack); } - mediaStream.AddTrack(new AudioStreamTrack("audioTrack")); + mediaStreams[0].AddTrack(new AudioStreamTrack("audioTrack")); Audio.Start(); signaling = new Signaling(urlSignaling); @@ -138,11 +141,16 @@ IEnumerator GetOffer() { continue; } - var pc = new RTCPeerConnection(); - pcs.Add(offer.connectionId, pc); + RTCConfiguration config = default; + config.iceServers = new RTCIceServer[] + { + new RTCIceServer { urls = urlsIceServer }, + }; + config.bundle_policy = RTCBundlePolicy.kBundlePolicyMaxBundle; + var pc = new RTCPeerConnection(ref config); + pcs.Add(offer.connectionId, pc); pc.OnDataChannel = new DelegateOnDataChannel(channel => { OnDataChannel(pc, channel); }); - pc.SetConfiguration(ref conf); pc.OnIceCandidate = new DelegateOnIceCandidate(candidate => { StartCoroutine(OnIceCandidate(offer.connectionId, candidate)); }); pc.OnIceConnectionChange = new DelegateOnIceConnectionChange(state => { @@ -152,13 +160,21 @@ IEnumerator GetOffer() pcs.Remove(offer.connectionId); } }); + //make video bit rate starts at 16000kbits, and 160000kbits at max. string pattern = @"(a=fmtp:\d+ .*level-asymmetry-allowed=.*)\r\n"; _desc.sdp = Regex.Replace(_desc.sdp, pattern, "$1;x-google-start-bitrate=16000;x-google-max-bitrate=160000\r\n"); + Debug.Log("remote sdp---------------------------------------------------------"); + Debug.Log(_desc.sdp); + pc.SetRemoteDescription(ref _desc); - foreach (var track in mediaStream.GetTracks()) + + foreach (var mediaStream in mediaStreams) { - pc.AddTrack(track); + foreach (var track in mediaStream.GetTracks()) + { + pc.AddTrack(track, mediaStream.Id); + } } StartCoroutine(Answer(connectionId)); @@ -178,12 +194,14 @@ IEnumerator Answer(string connectionId) } var opLocalDesc = pc.SetLocalDescription(ref op.desc); yield return opLocalDesc; + Debug.Log("local sdp---------------------------------------------------------"); + Debug.Log(op.desc.sdp); if (opLocalDesc.isError) { Debug.LogError($"Network Error: {opLocalDesc.error}"); yield break; } - var op3 = signaling.PostAnswer(this.sessionId, connectionId, op.desc.sdp); + var op3 = signaling.PostAnswer(this.sessionId, connectionId, op.desc.sdp); yield return op3; if (op3.webRequest.isNetworkError) { diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 47c3a27ff..300c72f33 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -9,16 +9,24 @@ namespace Unity.WebRTC public class MediaStream { internal IntPtr nativePtr; + internal string id; protected List mediaStreamTrackList = new List(); + private static int sMediaStreamCount = 0; + + public string Id { get => id; private set { } } public MediaStream() : base() { - nativePtr = WebRTC.Context.CreateMediaStream("MediaStream"); + sMediaStreamCount++; + id = "MediaStream" + sMediaStreamCount; + nativePtr = WebRTC.Context.CreateMediaStream(id); } public MediaStream(MediaStreamTrack[] tracks) : base() { - nativePtr = WebRTC.Context.CreateMediaStream("MediaStream"); + sMediaStreamCount++; + id = "MediaStream" + sMediaStreamCount; + nativePtr = WebRTC.Context.CreateMediaStream(id); foreach (var t in tracks) { @@ -93,7 +101,7 @@ public static void CreateRenderStreamTexture(this Camera cam, int width, int hei camRenderTexture.Create(); int mipCount = count; - for (int i = 1, mipLevel = 1; i <= mipCount; ++i, mipLevel *= 2) + for (int i = 1, mipLevel = 1; i <= mipCount; ++i, mipLevel *= 4) { RenderTexture webRtcTex = new RenderTexture(width / mipLevel, height / mipLevel, 0, RenderTextureFormat.BGRA32); webRtcTex.Create(); diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs index 2c08b378e..5d30853d7 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs @@ -36,7 +36,7 @@ private set { } public class VideoStreamTrack : MediaStreamTrack { - public VideoStreamTrack(string label, RenderTexture rt) : base(WebRTC.Context.CreateVideoTrack(label, rt.GetNativeTexturePtr(), rt.width, rt.height)) + public VideoStreamTrack(string label, RenderTexture rt, int bitRateMbps=10000000) : base(WebRTC.Context.CreateVideoTrack(label, rt.GetNativeTexturePtr(), rt.width, rt.height, bitRateMbps)) { } } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs index ee9b9b975..53392be65 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/RTCPeerConnection.cs @@ -188,9 +188,9 @@ public void Close() NativeMethods.PeerConnectionClose(self, m_id); } - public RTCRtpSender AddTrack(MediaStreamTrack track) + public RTCRtpSender AddTrack(MediaStreamTrack track, string mediaStreamId="unity") { - return new RTCRtpSender(NativeMethods.PeerConnectionAddTrack(self, track.nativePtr)); + return new RTCRtpSender(NativeMethods.PeerConnectionAddTrack(self, track.nativePtr, mediaStreamId)); } public void RemoveTrack(RTCRtpSender sender) diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs index 5ea60fa1b..5a2f6d266 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs @@ -178,11 +178,19 @@ public enum RTCIceTransportPolicy All } + public enum RTCBundlePolicy + { + kBundlePolicyBalanced, + kBundlePolicyMaxBundle, + kBundlePolicyMaxCompat + }; + [Serializable] public struct RTCConfiguration { public RTCIceServer[] iceServers; public RTCIceTransportPolicy iceTransportPolicy; + public RTCBundlePolicy bundle_policy; } public static class WebRTC @@ -326,7 +334,7 @@ internal static class NativeMethods [DllImport(WebRTC.Lib)] public static extern void PeerConnectionSetRemoteDescription(IntPtr ptr, ref RTCSessionDescription desc); [DllImport(WebRTC.Lib)] - public static extern IntPtr PeerConnectionAddTrack(IntPtr pc, IntPtr track); + public static extern IntPtr PeerConnectionAddTrack(IntPtr pc, IntPtr track, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string mediaStreamId); [DllImport(WebRTC.Lib)] public static extern void PeerConnectionRemoveTrack(IntPtr pc, IntPtr sender); [DllImport(WebRTC.Lib)] @@ -364,7 +372,7 @@ internal static class NativeMethods [DllImport(WebRTC.Lib)] public static extern IntPtr CreateMediaStream(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label); [DllImport(WebRTC.Lib)] - public static extern IntPtr CreateVideoTrack(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label, IntPtr rt, int width, int height); + public static extern IntPtr CreateVideoTrack(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label, IntPtr rt, int width, int height, int bitRate); [DllImport(WebRTC.Lib)] public static extern IntPtr CreateAudioTrack(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label); [DllImport(WebRTC.Lib)] @@ -404,7 +412,7 @@ internal struct Context public static Context Create(int uid = 0) { return NativeMethods.ContextCreate(uid); } public void Destroy(int uid = 0) { NativeMethods.ContextDestroy(uid); self = IntPtr.Zero; } public IntPtr CreateMediaStream(string label) { return NativeMethods.CreateMediaStream(self, label); } - public IntPtr CreateVideoTrack(string label, IntPtr rt, int width, int height) { return NativeMethods.CreateVideoTrack(self, label, rt, width, height); } + public IntPtr CreateVideoTrack(string label, IntPtr rt, int width, int height, int bitRate) { return NativeMethods.CreateVideoTrack(self, label, rt, width, height, bitRate); } public IntPtr CreateAudioTrack(string label) {return NativeMethods.CreateAudioTrack(self, label);} } } diff --git a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs index d42856a03..51b296ea0 100644 --- a/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs +++ b/Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs @@ -215,7 +215,8 @@ void Call() int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - mediaStream.AddTrack(new VideoStreamTrack("videoTrack"+1, cam.GetStreamTexture(i))); + RenderTexture rt = cam.GetStreamTexture(i); + mediaStream.AddTrack(new VideoStreamTrack("videoTrack"+1, rt)); } mediaStream.AddTrack(new AudioStreamTrack("audioTrack")); diff --git a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs index 9b0e2081c..dd391660b 100644 --- a/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs +++ b/Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs @@ -53,7 +53,8 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream() int texCount = cam.GetStreamTextureCount(); for (int i = 0; i < texCount; ++i) { - VideoStreamTrack videoStreamTrack = new VideoStreamTrack("videoTrack"+i,cam.GetStreamTexture(i)); + RenderTexture rt = cam.GetStreamTexture(i); + VideoStreamTrack videoStreamTrack = new VideoStreamTrack("videoTrack"+i, rt); mediaStream.AddTrack(videoStreamTrack); pc1Senders.Add(peer1.AddTrack(videoStreamTrack)); } diff --git a/Plugin/WebRTCPlugin/Context.cpp b/Plugin/WebRTCPlugin/Context.cpp index 11cdc906e..f1f770669 100644 --- a/Plugin/WebRTCPlugin/Context.cpp +++ b/Plugin/WebRTCPlugin/Context.cpp @@ -241,6 +241,9 @@ namespace WebRTC } config.servers.push_back(stunServer); config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan; + + Json::Value bundle_policy = configJson["bundle_policy"]; + config.bundle_policy = (webrtc::PeerConnectionInterface::BundlePolicy)bundle_policy.asInt(); } #pragma warning(push) #pragma warning(disable: 4715) @@ -313,6 +316,7 @@ namespace WebRTC Context::~Context() { + pDummyVideoEncoderFactory->Destroy(); clients.clear(); peerConnectionFactory = nullptr; @@ -324,9 +328,6 @@ namespace WebRTC workerThread.reset(); signalingThread->Quit(); signalingThread.reset(); - - pDummyVideoEncoderFactory->Destroy(); - } void Context::EncodeFrame() @@ -355,9 +356,9 @@ namespace WebRTC return mediaStreamMap[stream_id]; } - webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) + webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height, int32 bitRate) { - UnityEncoder* pUnityEncoder = pDummyVideoEncoderFactory->CreatePlatformEncoder(WebRTC::Nvidia, width, height); + UnityEncoder* pUnityEncoder = pDummyVideoEncoderFactory->CreatePlatformEncoder(WebRTC::Nvidia, width, height, bitRate); UnityVideoCapturer* pUnityVideoCapturer = new UnityVideoCapturer(pUnityEncoder, width, height); pUnityVideoCapturer->InitializeEncoder(); pDummyVideoEncoderFactory->AddCapturer(pUnityVideoCapturer); diff --git a/Plugin/WebRTCPlugin/Context.h b/Plugin/WebRTCPlugin/Context.h index aa4ba44fa..c1c5d7dff 100644 --- a/Plugin/WebRTCPlugin/Context.h +++ b/Plugin/WebRTCPlugin/Context.h @@ -44,7 +44,7 @@ namespace WebRTC public: explicit Context(int uid = -1); webrtc::MediaStreamInterface* CreateMediaStream(const std::string& stream_id); - webrtc::MediaStreamTrackInterface* CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height); + webrtc::MediaStreamTrackInterface* CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height, int32 bitRate); webrtc::MediaStreamTrackInterface* CreateAudioTrack(const std::string& label); ~Context(); diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp index 618a7435a..fe367d9ad 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp @@ -106,16 +106,16 @@ namespace WebRTC { //todo: According to condition of format choose different capturer. - UnityVideoCapturer* pCapturer = *capturers.begin(); + //UnityVideoCapturer* pCapturer = *(++capturers.begin()); - dummyVideoEncoder->SetKeyFrame.connect(pCapturer, &UnityVideoCapturer::SetKeyFrame); - dummyVideoEncoder->SetRate.connect(pCapturer, &UnityVideoCapturer::SetRate); + //dummyVideoEncoder->SetKeyFrame.connect(pCapturer, &UnityVideoCapturer::SetKeyFrame); + //dummyVideoEncoder->SetRate.connect(pCapturer, &UnityVideoCapturer::SetRate); } return dummyVideoEncoder; } - UnityEncoder* DummyVideoEncoderFactory::CreatePlatformEncoder(EncoderPlatform platform, int width, int height) + UnityEncoder* DummyVideoEncoderFactory::CreatePlatformEncoder(EncoderPlatform platform, int width, int height, int bitRate) { UnityEncoder* pEncoder = NULL; switch (platform) @@ -130,7 +130,7 @@ namespace WebRTC default: break; } - pEncoder->InitEncoder(width, height); + pEncoder->InitEncoder(width, height, bitRate); unityEncoders.push_back(pEncoder); return pEncoder; } diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.h b/Plugin/WebRTCPlugin/DummyVideoEncoder.h index c791fbe2b..c8d410365 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.h +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.h @@ -62,7 +62,7 @@ namespace WebRTC void Destroy(); void AddCapturer(UnityVideoCapturer* _capturer) { capturers.push_back(_capturer); } - UnityEncoder* CreatePlatformEncoder(EncoderPlatform platform, int width, int height); + UnityEncoder* CreatePlatformEncoder(EncoderPlatform platform, int width, int height, int bitRate); private: std::list capturers; std::list unityEncoders; diff --git a/Plugin/WebRTCPlugin/NvEncoder.cpp b/Plugin/WebRTCPlugin/NvEncoder.cpp index b7f737967..22623db8d 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.cpp +++ b/Plugin/WebRTCPlugin/NvEncoder.cpp @@ -39,10 +39,11 @@ namespace WebRTC } - void NvEncoder::InitEncoder(int width, int height) + void NvEncoder::InitEncoder(int width, int height, int _bitRate) { encodeWidth = width; encodeHeight = height; + bitRate = _bitRate; LogPrint(StringFormat("width is %d, height is %d", encodeWidth, encodeHeight).c_str()); checkf(g_D3D11Device != nullptr, "D3D11Device is invalid"); diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index 5761e597b..5510ad50c 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -39,7 +39,7 @@ namespace WebRTC bool IsSupported() const { return isNvEncoderSupported; } void SetIdrFrame() { isIdrFrame = true; } uint64 GetCurrentFrameCount() { return frameCount; } - void InitEncoder(int width, int height); + void InitEncoder(int width, int height, int _bitRate); void InitEncoderResources(); void* getRenderTexture() { return nvRenderTexture; } diff --git a/Plugin/WebRTCPlugin/UnityEncoder.h b/Plugin/WebRTCPlugin/UnityEncoder.h index 07bd0dcc5..127cf9528 100644 --- a/Plugin/WebRTCPlugin/UnityEncoder.h +++ b/Plugin/WebRTCPlugin/UnityEncoder.h @@ -16,7 +16,7 @@ namespace WebRTC virtual bool IsSupported() const = 0; virtual void SetIdrFrame() = 0; virtual uint64 GetCurrentFrameCount() = 0; - virtual void InitEncoder(int width, int height) = 0; + virtual void InitEncoder(int width, int height, int _bitRate) = 0; virtual void InitEncoderResources() = 0; virtual void* getRenderTexture() = 0; }; diff --git a/Plugin/WebRTCPlugin/WebRTCPlugin.cpp b/Plugin/WebRTCPlugin/WebRTCPlugin.cpp index 707c0d88d..71c141e3f 100644 --- a/Plugin/WebRTCPlugin/WebRTCPlugin.cpp +++ b/Plugin/WebRTCPlugin/WebRTCPlugin.cpp @@ -34,9 +34,9 @@ extern "C" return context->CreateMediaStream(label); } - UNITY_INTERFACE_EXPORT webrtc::MediaStreamTrackInterface* CreateVideoTrack(Context* context, const char* label, UnityFrameBuffer* frameBuffer, int32 width, int32 height) + UNITY_INTERFACE_EXPORT webrtc::MediaStreamTrackInterface* CreateVideoTrack(Context* context, const char* label, UnityFrameBuffer* frameBuffer, int32 width, int32 height, int32 bitRate) { - return context->CreateVideoTrack(label, frameBuffer, width, height); + return context->CreateVideoTrack(label, frameBuffer, width, height, bitRate); } UNITY_INTERFACE_EXPORT webrtc::MediaStreamTrackInterface* CreateAudioTrack(Context* context, const char* label) @@ -190,9 +190,9 @@ extern "C" obj->Close(); ContextManager::GetInstance()->curContext->DeleteClient(id); } - UNITY_INTERFACE_EXPORT webrtc::RtpSenderInterface* PeerConnectionAddTrack(PeerConnectionObject* obj, webrtc::MediaStreamTrackInterface* track) + UNITY_INTERFACE_EXPORT webrtc::RtpSenderInterface* PeerConnectionAddTrack(PeerConnectionObject* obj, webrtc::MediaStreamTrackInterface* track, const char* mediaStreamId) { - return obj->connection->AddTrack(rtc::scoped_refptr (track), { "unity" }).value().get(); + return obj->connection->AddTrack(rtc::scoped_refptr (track), { mediaStreamId }).value().get(); } UNITY_INTERFACE_EXPORT void PeerConnectionRemoveTrack(PeerConnectionObject* obj, webrtc::RtpSenderInterface* sender) diff --git a/WebApp/public/scripts/app.js b/WebApp/public/scripts/app.js index 200b276b1..282cf5ace 100644 --- a/WebApp/public/scripts/app.js +++ b/WebApp/public/scripts/app.js @@ -48,6 +48,16 @@ function onClickPlayButton() { sendClickEvent(videoPlayer, 2); }); + // add Switch Resolution button + const elementSwitchResolutionButton = document.createElement('button'); + elementSwitchResolutionButton.id = "switchResolutionButton"; + elementSwitchResolutionButton.innerHTML = "Switch Resolution"; + playerDiv.appendChild(elementSwitchResolutionButton); + elementSwitchResolutionButton.addEventListener ("click", function() { + videoPlayer.switchStream(); + }); + + // add fullscreen button const elementFullscreenButton = document.createElement('img'); elementFullscreenButton.id = 'fullscreenButton'; diff --git a/WebApp/public/scripts/video-player.js b/WebApp/public/scripts/video-player.js index 9723cd77d..19cfa337c 100644 --- a/WebApp/public/scripts/video-player.js +++ b/WebApp/public/scripts/video-player.js @@ -1,11 +1,14 @@ import Signaling from "./signaling.js" export class VideoPlayer { + constructor(element, config) { const _this = this; this.cfg = VideoPlayer.getConfiguration(config); this.pc = null; this.channel = null; + this.UnityStreams = []; + this.UnityStreamIndex = 0; this.offerOptions = { offerToReceiveAudio: true, offerToReceiveVideo: true, @@ -27,6 +30,7 @@ export class VideoPlayer { } config.sdpSemantics = 'unified-plan'; config.iceServers = [{urls: ['stun:stun.l.google.com:19302']}]; + config.bundlePolicy = "max-bundle"; return config; } @@ -48,6 +52,10 @@ export class VideoPlayer { // Create peerConnection with proxy server and set up handlers this.pc = new RTCPeerConnection(this.cfg); + this.pc.addTransceiver("video"); + this.pc.addTransceiver("audio"); + this.pc.addTransceiver("video"); + this.pc.onsignalingstatechange = function (e) { console.log('signalingState changed:', e); }; @@ -64,6 +72,10 @@ export class VideoPlayer { this.pc.ontrack = function (e) { console.log('New track added: ', e.streams); _this.video.srcObject = e.streams[0]; + if (_this.UnityStreams.indexOf(e.streams[0])==-1) + { + _this.UnityStreams.push(e.streams[0]); + } }; this.pc.onicecandidate = function (e) { if(e.candidate != null) { @@ -92,6 +104,7 @@ export class VideoPlayer { await this.createConnection(); // set local sdp offer.sdp = offer.sdp.replace(/useinbandfec=1/, 'useinbandfec=1;stereo=1;maxaveragebitrate=1048576'); + const desc = new RTCSessionDescription({sdp:offer.sdp, type:"offer"}); await this.pc.setLocalDescription(desc); await this.sendOffer(offer); @@ -162,6 +175,15 @@ export class VideoPlayer { } }; + switchStream(){ + this.video.srcObject = this.UnityStreams[this.UnityStreamIndex]; + this.UnityStreamIndex++; + if (this.UnityStreamIndex>=this.UnityStreams.length) + { + this.UnityStreamIndex = 0; + } + }; + sendMsg(msg) { if(this.channel == null) { return; diff --git a/WebApp/public/stylesheets/style.css b/WebApp/public/stylesheets/style.css index 4ff845946..b2a12c822 100644 --- a/WebApp/public/stylesheets/style.css +++ b/WebApp/public/stylesheets/style.css @@ -57,10 +57,25 @@ body{ font-size: 16px; } +#switchResolutionButton{ + position: absolute; + bottom: 10px; + left: 380px; + width: 160px; + background-color: #FF6C0B; /* Orange */ + border: none; + color: white; + padding: 15px 32px; + text-align: center; + text-decoration: none; + display: inline-block; + font-size: 16px; +} + #fullscreenButton{ position: absolute; top: 25px; right: 25px; width: 32px; height: 32px; -} \ No newline at end of file +} From f56b2cae2e3891283b3cd7ce2837fb7b46f70058 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 12 Aug 2019 15:22:28 +0800 Subject: [PATCH 24/27] same resolution nvEncoders share input texture. --- Assets/Scripts/RenderStreaming.cs | 53 +++++++++----- .../Runtime/Srcipts/MediaStream.cs | 61 ++++++++-------- .../Runtime/Srcipts/WebRTC.cs | 14 ++-- Plugin/WebRTCPlugin/DummyVideoEncoder.cpp | 13 ++++ Plugin/WebRTCPlugin/DummyVideoEncoder.h | 1 + Plugin/WebRTCPlugin/NvEncoder.cpp | 69 ++++++++++++++----- Plugin/WebRTCPlugin/NvEncoder.h | 42 +++++++++-- Plugin/WebRTCPlugin/UnityEncoder.h | 7 +- Plugin/WebRTCPlugin/UnityVideoCapturer.cpp | 3 +- WebApp/public/scripts/video-player.js | 2 + 10 files changed, 186 insertions(+), 79 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index eb1ceaab3..bff7a8c36 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -17,6 +17,12 @@ public class ButtonClickElement public ButtonClickEvent click; } + public class CameraMediaStream + { + public Camera camera; + public MediaStream[] mediaStreams = new MediaStream[2]; + } + public class RenderStreaming : MonoBehaviour { #pragma warning disable 0649 @@ -29,9 +35,6 @@ public class RenderStreaming : MonoBehaviour [SerializeField, Tooltip("Time interval for polling from signaling server")] private float interval = 5.0f; - [SerializeField, Tooltip("Camera to capture video stream")] - private Camera captureCamera; - [SerializeField] private ButtonClickElement[] arrayButtonClickEvent; #pragma warning restore 0649 @@ -41,7 +44,7 @@ public class RenderStreaming : MonoBehaviour private Dictionary> mapChannels = new Dictionary>(); private RTCConfiguration conf; private string sessionId; - private MediaStream[] mediaStreams = new MediaStream[2]; + private Dictionary cameraMediaStreamDict = new Dictionary(); public void Awake() { @@ -62,19 +65,30 @@ public IEnumerator Start() yield break; } - captureCamera.CreateRenderStreamTexture(1280, 720, mediaStreams.Length); - - int texCount = captureCamera.GetStreamTextureCount(); - for (int i = 0; i < texCount; ++i) + int count = 0; + foreach (var camera in Camera.allCameras) { - int index = i; - mediaStreams[i] = new MediaStream(); - RenderTexture rt = captureCamera.GetStreamTexture(index); - VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, rt); - mediaStreams[i].AddTrack(videoTrack); - } + count++; + if (count == 1) + { + //continue; + } - mediaStreams[0].AddTrack(new AudioStreamTrack("audioTrack")); + CameraMediaStream cameraMediaStream = new CameraMediaStream(); + cameraMediaStreamDict.Add(camera, cameraMediaStream); + camera.CreateRenderStreamTexture(1280, 720, cameraMediaStream.mediaStreams.Length); + int texCount = camera.GetStreamTextureCount(); + for (int i = 0; i < texCount; ++i) + { + int index = i; + cameraMediaStream.mediaStreams[i] = new MediaStream(); + RenderTexture rt = camera.GetStreamTexture(index); + VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, rt); + cameraMediaStream.mediaStreams[i].AddTrack(videoTrack); + cameraMediaStream.mediaStreams[i].AddTrack(new AudioStreamTrack("audioTrack")); + } + } + Audio.Start(); signaling = new Signaling(urlSignaling); @@ -169,11 +183,14 @@ IEnumerator GetOffer() pc.SetRemoteDescription(ref _desc); - foreach (var mediaStream in mediaStreams) + foreach (var k in cameraMediaStreamDict.Keys) { - foreach (var track in mediaStream.GetTracks()) + foreach (var mediaStream in cameraMediaStreamDict[k].mediaStreams) { - pc.AddTrack(track, mediaStream.Id); + foreach (var track in mediaStream.GetTracks()) + { + pc.AddTrack(track, mediaStream.Id); + } } } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs index 300c72f33..e9fa3bf44 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs @@ -74,63 +74,68 @@ public static void AddCleanerCallback(this GameObject obj, Action callback) Cleaner.AddCleanerCallback(obj, callback); } } + + internal class CameraCapturerTextures + { + internal RenderTexture camRenderTexture; + internal List webRTCTextures = new List(); + } + public static class CameraExtension { - internal static RenderTexture camRenderTexture; - internal static List webRTCTextures = new List(); - internal static List camCopyRts = new List(); - internal static bool started = false; + internal static Dictionary camCapturerTexturesDict = new Dictionary(); public static int GetStreamTextureCount(this Camera cam) { - return webRTCTextures.Count; + CameraCapturerTextures textures; + if (camCapturerTexturesDict.TryGetValue(cam, out textures)) + { + return textures.webRTCTextures.Count; + } + return 0; } public static RenderTexture GetStreamTexture(this Camera cam, int index) { - return webRTCTextures[index]; + CameraCapturerTextures textures; + if (camCapturerTexturesDict.TryGetValue(cam, out textures)) + { + if (index >= 0 && index < textures.webRTCTextures.Count) + { + return textures.webRTCTextures[index]; + } + } + return null; } public static void CreateRenderStreamTexture(this Camera cam, int width, int height, int count = 1) { - if (camCopyRts.Count > 0) - { - throw new NotImplementedException("Currently not allowed multiple MediaStream"); - } + CameraCapturerTextures cameraCapturerTextures = new CameraCapturerTextures(); + camCapturerTexturesDict.Add(cam, cameraCapturerTextures); - camRenderTexture = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32); - camRenderTexture.Create(); + cameraCapturerTextures.camRenderTexture = new RenderTexture(width, height, 0, RenderTextureFormat.BGRA32); + cameraCapturerTextures.camRenderTexture.Create(); int mipCount = count; for (int i = 1, mipLevel = 1; i <= mipCount; ++i, mipLevel *= 4) { RenderTexture webRtcTex = new RenderTexture(width / mipLevel, height / mipLevel, 0, RenderTextureFormat.BGRA32); webRtcTex.Create(); - webRTCTextures.Add(webRtcTex); + cameraCapturerTextures.webRTCTextures.Add(webRtcTex); } - cam.targetTexture = camRenderTexture; + cam.targetTexture = cameraCapturerTextures.camRenderTexture; cam.gameObject.AddCleanerCallback(() => { - camRenderTexture.Release(); - UnityEngine.Object.Destroy(camRenderTexture); + cameraCapturerTextures.camRenderTexture.Release(); + UnityEngine.Object.Destroy(cameraCapturerTextures.camRenderTexture); - foreach (var v in webRTCTextures) + foreach (var v in cameraCapturerTextures.webRTCTextures) { v.Release(); UnityEngine.Object.Destroy(v); } - webRTCTextures.Clear(); + cameraCapturerTextures.webRTCTextures.Clear(); }); - started = true; - } - - public static void RemoveRt(RenderTexture[] rts) - { - camCopyRts.Remove(rts); - if (camCopyRts.Count == 0) - { - started = false; - } } } diff --git a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs index 5a2f6d266..809d702af 100644 --- a/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs +++ b/Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs @@ -227,15 +227,17 @@ public static IEnumerator Update() { // Wait until all frame rendering is done yield return new WaitForEndOfFrame(); - if (CameraExtension.started) + //Blit is for DirectX Rendering API Only + + foreach (var k in CameraExtension.camCapturerTexturesDict.Keys) { - //Blit is for DirectX Rendering API Only - foreach(var rt in CameraExtension.webRTCTextures) + foreach (var rt in CameraExtension.camCapturerTexturesDict[k].webRTCTextures) { - Graphics.Blit(CameraExtension.camRenderTexture, rt, flipMat); - } - GL.IssuePluginEvent(NativeMethods.GetRenderEventFunc(), 0); + Graphics.Blit(CameraExtension.camCapturerTexturesDict[k].camRenderTexture, rt, flipMat); + } } + + GL.IssuePluginEvent(NativeMethods.GetRenderEventFunc(), 0); Audio.Update(); } } diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp index fe367d9ad..7cba0071d 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.cpp @@ -82,6 +82,7 @@ namespace WebRTC delete *it; } unityEncoders.clear(); + NvEncoder::DestroyEncoderTexture(); } std::vector DummyVideoEncoderFactory::GetSupportedFormats() const @@ -135,4 +136,16 @@ namespace WebRTC return pEncoder; } + UnityEncoder* DummyVideoEncoderFactory::GetPlatformEncoder(EncoderPlatform platform, int width, int height, int bitRate) + { + for (std::list::iterator it = unityEncoders.begin(); it != unityEncoders.end(); ++it) + { + if ((*it)->getEncodeWidth() == width && (*it)->getEncodeHeight() == height && (*it)->getBitRate() == bitRate) { + return (*it); + } + } + + return CreatePlatformEncoder(platform, width, height, bitRate); + } + } diff --git a/Plugin/WebRTCPlugin/DummyVideoEncoder.h b/Plugin/WebRTCPlugin/DummyVideoEncoder.h index c8d410365..fe20463ec 100644 --- a/Plugin/WebRTCPlugin/DummyVideoEncoder.h +++ b/Plugin/WebRTCPlugin/DummyVideoEncoder.h @@ -63,6 +63,7 @@ namespace WebRTC void AddCapturer(UnityVideoCapturer* _capturer) { capturers.push_back(_capturer); } UnityEncoder* CreatePlatformEncoder(EncoderPlatform platform, int width, int height, int bitRate); + UnityEncoder* GetPlatformEncoder(EncoderPlatform platform, int width, int height, int bitRate); private: std::list capturers; std::list unityEncoders; diff --git a/Plugin/WebRTCPlugin/NvEncoder.cpp b/Plugin/WebRTCPlugin/NvEncoder.cpp index 22623db8d..bc805c406 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.cpp +++ b/Plugin/WebRTCPlugin/NvEncoder.cpp @@ -6,7 +6,7 @@ namespace WebRTC { - + std::list NvEncoder::nvEncoderInputTextureList; NvEncoder::NvEncoder() { if (pEncoderInterface==nullptr) @@ -137,11 +137,13 @@ namespace WebRTC lastBitRate = bitRate; } } + //entry for encoding a frame void NvEncoder::EncodeFrame(int width, int height) { UpdateSettings(width, height); uint32 bufferIndexToWrite = frameCount % bufferedFrameNum; + Frame& frame = bufferedFrames[bufferIndexToWrite]; #pragma region set frame params //no free buffer, skip this frame @@ -168,13 +170,15 @@ namespace WebRTC picParams.encodePicFlags |= NV_ENC_PIC_FLAG_FORCEIDR; } isIdrFrame = false; + bool result = NV_RESULT((errorCode = ContextManager::GetInstance()->pNvEncodeAPI->nvEncEncodePicture(pEncoderInterface, &picParams))); checkf(result, StringFormat("Failed to encode frame, error is %d", errorCode).c_str()); + #pragma endregion ProcessEncodedFrame(frame); frameCount++; } - + //get encoded frame void NvEncoder::ProcessEncodedFrame(Frame& frame) { @@ -183,12 +187,15 @@ namespace WebRTC { return; } + frame.isEncoding = false; + #pragma region retrieve encoded frame from output buffer NV_ENC_LOCK_BITSTREAM lockBitStream = { 0 }; lockBitStream.version = NV_ENC_LOCK_BITSTREAM_VER; lockBitStream.outputBitstream = frame.outputFrame; lockBitStream.doNotWait = nvEncInitializeParams.enableEncodeAsync; + bool result = NV_RESULT((errorCode = ContextManager::GetInstance()->pNvEncodeAPI->nvEncLockBitstream(pEncoderInterface, &lockBitStream))); checkf(result, StringFormat("Failed to lock bit stream, error is %d", errorCode).c_str()); if (lockBitStream.bitstreamSizeInBytes) @@ -196,12 +203,11 @@ namespace WebRTC frame.encodedFrame.resize(lockBitStream.bitstreamSizeInBytes); std::memcpy(frame.encodedFrame.data(), lockBitStream.bitstreamBufferPtr, lockBitStream.bitstreamSizeInBytes); } - result = NV_RESULT((errorCode = ContextManager::GetInstance()->pNvEncodeAPI->nvEncUnlockBitstream(pEncoderInterface, frame.outputFrame))); checkf(result, StringFormat("Failed to unlock bit stream, error is %d", errorCode).c_str()); frame.isIdrFrame = lockBitStream.pictureType == NV_ENC_PIC_TYPE_IDR; #pragma endregion - CaptureFrame(frame.encodedFrame); + captureFrame(frame.encodedFrame); } ID3D11Texture2D* NvEncoder::AllocateInputBuffers() @@ -254,15 +260,42 @@ namespace WebRTC StringFormat("nvEncCreateBitstreamBuffer error is %d", errorCode).c_str()); return createBitstreamBuffer.bitstreamBuffer; } - void NvEncoder::InitEncoderResources() + + void NvEncoder::DestroyEncoderTexture() + { + for (std::list::iterator it = nvEncoderInputTextureList.begin(); it != nvEncoderInputTextureList.end(); ++it) + { + delete (*it); + } + nvEncoderInputTextureList.clear(); + } + + UnityFrameBuffer* NvEncoder::getEncoderTexture(int width, int height) { - nvRenderTexture = AllocateInputBuffers(); - Frame& frame = bufferedFrames[0]; - frame.inputFrame.registeredResource = RegisterResource(nvRenderTexture); - frame.inputFrame.bufferFormat = NV_ENC_BUFFER_FORMAT_ARGB; - MapResources(frame.inputFrame); - frame.outputFrame = InitializeBitstreamBuffer(); + for (std::list::iterator it = nvEncoderInputTextureList.begin(); it!= nvEncoderInputTextureList.end(); ++it) + { + if ( (*it)->width==width && (*it)->height==height ) + { + return (*it)->texture; + } + } + EncoderInputTexture* pEncoderInputTexture = new EncoderInputTexture(width, height); + nvEncoderInputTextureList.push_back(pEncoderInputTexture); + return pEncoderInputTexture->texture; + } + + void NvEncoder::InitEncoderResources() + { + nvEncoderTexture = getEncoderTexture(encodeWidth, encodeHeight); + for (int i = 0; i < bufferedFrameNum; i++) + { + Frame& frame = bufferedFrames[i]; + frame.inputFrame.registeredResource = RegisterResource(nvEncoderTexture); + frame.inputFrame.bufferFormat = NV_ENC_BUFFER_FORMAT_ARGB; + MapResources(frame.inputFrame); + frame.outputFrame = InitializeBitstreamBuffer(); + } } void NvEncoder::ReleaseFrameInputBuffer(Frame& frame) { @@ -278,14 +311,14 @@ namespace WebRTC { for (Frame& frame : bufferedFrames) { - ReleaseFrameInputBuffer(frame); - bool result = NV_RESULT(ContextManager::GetInstance()->pNvEncodeAPI->nvEncDestroyBitstreamBuffer(pEncoderInterface, frame.outputFrame)); - checkf(result, "Failed to destroy output buffer bit stream"); - frame.outputFrame = nullptr; + if (frame.outputFrame!=nullptr) + { + ReleaseFrameInputBuffer(frame); + bool result = NV_RESULT(ContextManager::GetInstance()->pNvEncodeAPI->nvEncDestroyBitstreamBuffer(pEncoderInterface, frame.outputFrame)); + checkf(result, "Failed to destroy output buffer bit stream"); + frame.outputFrame = nullptr; + } } - - nvRenderTexture->Release(); - nvRenderTexture = nullptr; } } diff --git a/Plugin/WebRTCPlugin/NvEncoder.h b/Plugin/WebRTCPlugin/NvEncoder.h index 5510ad50c..6a119765b 100644 --- a/Plugin/WebRTCPlugin/NvEncoder.h +++ b/Plugin/WebRTCPlugin/NvEncoder.h @@ -29,6 +29,35 @@ namespace WebRTC std::atomic isEncoding = false; }; + struct EncoderInputTexture + { + UnityFrameBuffer* texture; + int width; + int height; + EncoderInputTexture(int w, int h) + { + width = w; + height = h; + D3D11_TEXTURE2D_DESC desc = { 0 }; + desc.Width = width; + desc.Height = height; + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.BindFlags = D3D11_BIND_RENDER_TARGET; + desc.CPUAccessFlags = 0; + HRESULT r = g_D3D11Device->CreateTexture2D(&desc, NULL, &texture); + } + + ~EncoderInputTexture() + { + texture->Release(); + texture = nullptr; + } + }; + public: NvEncoder(); ~NvEncoder(); @@ -41,8 +70,13 @@ namespace WebRTC uint64 GetCurrentFrameCount() { return frameCount; } void InitEncoder(int width, int height, int _bitRate); void InitEncoderResources(); - void* getRenderTexture() { return nvRenderTexture; } - + void* getRenderTexture() { return nvEncoderTexture; } + int getEncodeWidth() { return encodeWidth; } + int getEncodeHeight() { return encodeHeight; } + int getBitRate() { return bitRate; } + static void DestroyEncoderTexture(); + private: + static UnityFrameBuffer* getEncoderTexture(int width, int height); private: void ReleaseFrameInputBuffer(Frame& frame); void ReleaseEncoderResources(); @@ -55,6 +89,8 @@ namespace WebRTC NV_ENC_CONFIG nvEncConfig = {}; _NVENCSTATUS errorCode; Frame bufferedFrames[bufferedFrameNum]; + static std::list nvEncoderInputTextureList; + UnityFrameBuffer* nvEncoderTexture; uint64 frameCount = 0; void* pEncoderInterface = nullptr; bool isNvEncoderSupported = false; @@ -69,8 +105,6 @@ namespace WebRTC //5Mbps const int minBitRate = 5000000; int frameRate = 45; - - UnityFrameBuffer* nvRenderTexture; }; } diff --git a/Plugin/WebRTCPlugin/UnityEncoder.h b/Plugin/WebRTCPlugin/UnityEncoder.h index 127cf9528..ecc80a058 100644 --- a/Plugin/WebRTCPlugin/UnityEncoder.h +++ b/Plugin/WebRTCPlugin/UnityEncoder.h @@ -2,14 +2,12 @@ namespace WebRTC { - class UnityEncoder { public: UnityEncoder(); virtual ~UnityEncoder(); - - sigslot::signal1&> CaptureFrame; + sigslot::signal1&> captureFrame; virtual void SetRate(uint32 rate) = 0; virtual void UpdateSettings(int width, int height) = 0; virtual void EncodeFrame(int width, int height) = 0; @@ -19,6 +17,9 @@ namespace WebRTC virtual void InitEncoder(int width, int height, int _bitRate) = 0; virtual void InitEncoderResources() = 0; virtual void* getRenderTexture() = 0; + virtual int getEncodeWidth() = 0; + virtual int getEncodeHeight() = 0; + virtual int getBitRate() = 0; }; } diff --git a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp index c980679cd..2383a7897 100644 --- a/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp +++ b/Plugin/WebRTCPlugin/UnityVideoCapturer.cpp @@ -40,7 +40,6 @@ namespace WebRTC void UnityVideoCapturer::InitializeEncoder() { - //koseyile todo: one nvEncoder can connect multiple capturers. - nvEncoder->CaptureFrame.connect(this, &UnityVideoCapturer::CaptureFrame); + nvEncoder->captureFrame.connect(this, &UnityVideoCapturer::CaptureFrame); } } diff --git a/WebApp/public/scripts/video-player.js b/WebApp/public/scripts/video-player.js index 19cfa337c..6432bb497 100644 --- a/WebApp/public/scripts/video-player.js +++ b/WebApp/public/scripts/video-player.js @@ -52,9 +52,11 @@ export class VideoPlayer { // Create peerConnection with proxy server and set up handlers this.pc = new RTCPeerConnection(this.cfg); + this.pc.addTransceiver("video"); this.pc.addTransceiver("audio"); this.pc.addTransceiver("video"); + this.pc.addTransceiver("audio"); this.pc.onsignalingstatechange = function (e) { console.log('signalingState changed:', e); From c9edf213bfa1ed0b03f760af42d9bfc01e7c70c4 Mon Sep 17 00:00:00 2001 From: chenyuan Date: Mon, 12 Aug 2019 18:41:07 +0800 Subject: [PATCH 25/27] add other way to implement simulcast --- Assets/Scripts/RenderStreaming.cs | 53 ++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/Assets/Scripts/RenderStreaming.cs b/Assets/Scripts/RenderStreaming.cs index bff7a8c36..fae0a69c2 100644 --- a/Assets/Scripts/RenderStreaming.cs +++ b/Assets/Scripts/RenderStreaming.cs @@ -37,6 +37,9 @@ public class RenderStreaming : MonoBehaviour [SerializeField] private ButtonClickElement[] arrayButtonClickEvent; + + [SerializeField] + private bool isUseMinimalTextures = true; #pragma warning restore 0649 private Signaling signaling; @@ -65,30 +68,44 @@ public IEnumerator Start() yield break; } - int count = 0; - foreach (var camera in Camera.allCameras) + + if (isUseMinimalTextures) { - count++; - if (count == 1) + foreach (var camera in Camera.allCameras) { - //continue; + CameraMediaStream cameraMediaStream = new CameraMediaStream(); + cameraMediaStreamDict.Add(camera, cameraMediaStream); + camera.CreateRenderStreamTexture(1280, 720); + int mediaCount = cameraMediaStream.mediaStreams.Length; + for (int i = 0; i < mediaCount; ++i) + { + cameraMediaStream.mediaStreams[i] = new MediaStream(); + RenderTexture rt = camera.GetStreamTexture(0); + int temp = i==0 ? 1 : (int)Mathf.Pow(i + 1, 10); + VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, rt, 1000000/temp); + cameraMediaStream.mediaStreams[i].AddTrack(videoTrack); + cameraMediaStream.mediaStreams[i].AddTrack(new AudioStreamTrack("audioTrack")); + } } - - CameraMediaStream cameraMediaStream = new CameraMediaStream(); - cameraMediaStreamDict.Add(camera, cameraMediaStream); - camera.CreateRenderStreamTexture(1280, 720, cameraMediaStream.mediaStreams.Length); - int texCount = camera.GetStreamTextureCount(); - for (int i = 0; i < texCount; ++i) + }else{ + foreach (var camera in Camera.allCameras) { - int index = i; - cameraMediaStream.mediaStreams[i] = new MediaStream(); - RenderTexture rt = camera.GetStreamTexture(index); - VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, rt); - cameraMediaStream.mediaStreams[i].AddTrack(videoTrack); - cameraMediaStream.mediaStreams[i].AddTrack(new AudioStreamTrack("audioTrack")); + CameraMediaStream cameraMediaStream = new CameraMediaStream(); + cameraMediaStreamDict.Add(camera, cameraMediaStream); + camera.CreateRenderStreamTexture(1280, 720, cameraMediaStream.mediaStreams.Length); + int texCount = camera.GetStreamTextureCount(); + for (int i = 0; i < texCount; ++i) + { + int index = i; + cameraMediaStream.mediaStreams[i] = new MediaStream(); + RenderTexture rt = camera.GetStreamTexture(index); + VideoStreamTrack videoTrack = new VideoStreamTrack("videoTrack" + i, rt); + cameraMediaStream.mediaStreams[i].AddTrack(videoTrack); + cameraMediaStream.mediaStreams[i].AddTrack(new AudioStreamTrack("audioTrack")); + } } } - + Audio.Start(); signaling = new Signaling(urlSignaling); From 9c8562075ec4ce40960ed49ea060b4a041fa96ec Mon Sep 17 00:00:00 2001 From: chenyuan Date: Thu, 15 Aug 2019 19:47:36 +0800 Subject: [PATCH 26/27] add select media stream ui. --- WebApp/public/scripts/app.js | 120 +++++++++++++++++++++++--- WebApp/public/scripts/video-player.js | 31 +++++++ WebApp/public/stylesheets/style.css | 82 ++++++++++++++---- 3 files changed, 208 insertions(+), 25 deletions(-) diff --git a/WebApp/public/scripts/app.js b/WebApp/public/scripts/app.js index 282cf5ace..f2784f998 100644 --- a/WebApp/public/scripts/app.js +++ b/WebApp/public/scripts/app.js @@ -48,16 +48,6 @@ function onClickPlayButton() { sendClickEvent(videoPlayer, 2); }); - // add Switch Resolution button - const elementSwitchResolutionButton = document.createElement('button'); - elementSwitchResolutionButton.id = "switchResolutionButton"; - elementSwitchResolutionButton.innerHTML = "Switch Resolution"; - playerDiv.appendChild(elementSwitchResolutionButton); - elementSwitchResolutionButton.addEventListener ("click", function() { - videoPlayer.switchStream(); - }); - - // add fullscreen button const elementFullscreenButton = document.createElement('img'); elementFullscreenButton.id = 'fullscreenButton'; @@ -84,6 +74,104 @@ function onClickPlayButton() { elementFullscreenButton.style.display = 'block'; } } + +} + +function setupMediaSelector(options) +{ + const playerDiv = document.getElementById('player'); + let mediaSelectDiv = document.createElement("div"); + mediaSelectDiv.id = "mediaSelect"; + mediaSelectDiv.setAttribute("style", "width:200px;"); + mediaSelectDiv.className = "custom-select"; + playerDiv.appendChild(mediaSelectDiv); + const mediaSelect = document.createElement("select"); + mediaSelectDiv.appendChild(mediaSelect); + let index = 0; + options.forEach(option=>{ + let optionItem = document.createElement("Option"); + optionItem.value = index++; + optionItem.innerHTML = option; + mediaSelect.appendChild(optionItem); + }) + + + let customSelects, selElmnt; + /*look for any elements with the class "custom-select":*/ + customSelects = document.getElementsByClassName("custom-select"); + for (let i = 0; i < customSelects.length; i++) { + selElmnt = customSelects[i].getElementsByTagName("select")[0]; + /*for each element, create a new DIV that will act as the selected item:*/ + let a = document.createElement("DIV"); + a.setAttribute("class", "select-selected"); + a.innerHTML = selElmnt.options[selElmnt.selectedIndex].innerHTML; + customSelects[i].appendChild(a); + /*for each element, create a new DIV that will contain the option list:*/ + let b = document.createElement("DIV"); + b.setAttribute("class", "select-items select-hide"); + for (let j = 1; j < selElmnt.length; j++) { + /*for each option in the original select element, + create a new DIV that will act as an option item:*/ + let c = document.createElement("DIV"); + c.innerHTML = selElmnt.options[j].innerHTML; + c.addEventListener("click", function(e) { + /*when an item is clicked, update the original select box, + and the selected item:*/ + let y, i, k, s, h; + s = this.parentNode.parentNode.getElementsByTagName("select")[0]; + + videoPlayer.selectMediaStream(this.innerHTML); + console.log(this.innerHTML); + + h = this.parentNode.previousSibling; + for (i = 0; i < s.length; i++) { + if (s.options[i].innerHTML == this.innerHTML) { + s.selectedIndex = i; + h.innerHTML = this.innerHTML; + y = this.parentNode.getElementsByClassName("same-as-selected"); + for (k = 0; k < y.length; k++) { + y[k].removeAttribute("class"); + } + this.setAttribute("class", "same-as-selected"); + break; + } + } + h.click(); + }); + b.appendChild(c); + } + customSelects[i].appendChild(b); + a.addEventListener("click", function(e) { + /*when the select box is clicked, close any other select boxes, + and open/close the current select box:*/ + e.stopPropagation(); + closeAllSelect(this); + this.nextSibling.classList.toggle("select-hide"); + this.classList.toggle("select-arrow-active"); + }); + } + function closeAllSelect(elmnt) { + /*a function that will close all select boxes in the document, + except the current select box:*/ + var x, y, i, arrNo = []; + x = document.getElementsByClassName("select-items"); + y = document.getElementsByClassName("select-selected"); + for (i = 0; i < y.length; i++) { + if (elmnt == y[i]) { + arrNo.push(i) + } else { + y[i].classList.remove("select-arrow-active"); + } + } + for (i = 0; i < x.length; i++) { + if (arrNo.indexOf(i)) { + x[i].classList.add("select-hide"); + } + } + } + /*if the user clicks anywhere outside the select box, + then close all select boxes:*/ + document.addEventListener("click", closeAllSelect); } async function setupVideoPlayer(element, config) { @@ -91,6 +179,7 @@ async function setupVideoPlayer(element, config) { await videoPlayer.setupConnection(); videoPlayer.ondisconnect = onDisconnect; + videoPlayer.onaddtrackfinish = onAddTrackFinish; registerKeyboardEvents(videoPlayer); registerMouseEvents(videoPlayer, element); @@ -104,6 +193,17 @@ function onDisconnect() { showPlayButton(); } +function onAddTrackFinish(mediaStreams) { + + let options = ["Select a media"]; + for (let i=0; i new Promise(resolve => setTimeout(resolve, msec)); } @@ -74,11 +76,30 @@ export class VideoPlayer { this.pc.ontrack = function (e) { console.log('New track added: ', e.streams); _this.video.srcObject = e.streams[0]; + + if (_this.UnityStreams.indexOf(e.streams[0])==-1) { + let videoTracks = e.streams[0].getVideoTracks(); + for(let i=0; i Date: Mon, 19 Aug 2019 17:32:51 +0800 Subject: [PATCH 27/27] web app can select media stream. --- WebApp/public/scripts/app.js | 28 +++++++++---- WebApp/public/scripts/video-player.js | 58 +++++++++++++-------------- WebApp/public/stylesheets/style.css | 20 ++++++--- 3 files changed, 61 insertions(+), 45 deletions(-) diff --git a/WebApp/public/scripts/app.js b/WebApp/public/scripts/app.js index f2784f998..ff55c0bd3 100644 --- a/WebApp/public/scripts/app.js +++ b/WebApp/public/scripts/app.js @@ -24,11 +24,20 @@ function onClickPlayButton() { const playerDiv = document.getElementById('player'); // add video player - const elementVideo = document.createElement('video'); - elementVideo.id = 'Video'; - elementVideo.style.touchAction = 'none'; - playerDiv.appendChild(elementVideo); - setupVideoPlayer(elementVideo).then(value => videoPlayer = value); + let elementVideos = []; + for (let i=0; i<2; i++) + { + const elementVideo = document.createElement('video'); + elementVideo.id = "Video"+i; + elementVideo.style.touchAction = 'none'; + playerDiv.appendChild(elementVideo); + + elementVideos.push(elementVideo); + } + + + setupVideoPlayer(elementVideos).then(value => videoPlayer = value); + // add green button const elementBlueButton = document.createElement('button'); @@ -174,14 +183,17 @@ function setupMediaSelector(options) document.addEventListener("click", closeAllSelect); } -async function setupVideoPlayer(element, config) { - const videoPlayer = new VideoPlayer(element, config); +async function setupVideoPlayer(elements, config) { + const videoPlayer = new VideoPlayer(elements, config); await videoPlayer.setupConnection(); videoPlayer.ondisconnect = onDisconnect; videoPlayer.onaddtrackfinish = onAddTrackFinish; registerKeyboardEvents(videoPlayer); - registerMouseEvents(videoPlayer, element); + + elements.forEach(element=>{ + registerMouseEvents(videoPlayer, element); + }); return videoPlayer; } diff --git a/WebApp/public/scripts/video-player.js b/WebApp/public/scripts/video-player.js index 4faa113b6..d8cbef707 100644 --- a/WebApp/public/scripts/video-player.js +++ b/WebApp/public/scripts/video-player.js @@ -2,23 +2,25 @@ import Signaling from "./signaling.js" export class VideoPlayer { - constructor(element, config) { + constructor(elements, config) { const _this = this; this.cfg = VideoPlayer.getConfiguration(config); this.pc = null; this.channel = null; this.UnityStreams = []; - this.UnityStreamIndex = 0; this.UnityStreamCount = 2; this.offerOptions = { offerToReceiveAudio: true, offerToReceiveVideo: true, }; - this.video = element; - this.video.playsInline = true; - this.video.addEventListener('loadedmetadata', function () { - _this.video.play(); - }, true); + this.videos = elements; + this.videos.forEach(v=>{ + v.playsInline = true; + v.addEventListener('loadedmetadata', function () { + v.play(); + }, true); + }) + this.interval = 3000; this.signaling = new Signaling(); this.ondisconnect = function(){}; @@ -60,6 +62,7 @@ export class VideoPlayer { this.pc.addTransceiver("video"); this.pc.addTransceiver("audio"); + this.pc.onsignalingstatechange = function (e) { console.log('signalingState changed:', e); }; @@ -73,20 +76,15 @@ export class VideoPlayer { this.pc.onicegatheringstatechange = function (e) { console.log('iceGatheringState changed:', e); }; + let tempCount = 0; this.pc.ontrack = function (e) { - console.log('New track added: ', e.streams); - _this.video.srcObject = e.streams[0]; + console.log('New track added: ', e.streams); + console.log(e.track); if (_this.UnityStreams.indexOf(e.streams[0])==-1) { - let videoTracks = e.streams[0].getVideoTracks(); - for(let i=0; i