Skip to content

Commit e400b81

Browse files
koseyilekoseyile
committed
decoupling streams and tracks. delete these functions: CaptureVideoStream CaptureAudioStream
1 parent f3972bd commit e400b81

File tree

9 files changed

+68
-94
lines changed

9 files changed

+68
-94
lines changed

‎Assets/Scripts/RenderStreaming.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ public IEnumerator Start()
6363
}
6464

6565
captureCamera.CreateRenderStreamTexture(1280, 720);
66-
mediaStream = new MediaStream();
66+
mediaStream = new MediaStream("MediaStream");
6767
int texCount = captureCamera.GetStreamTextureCount();
6868
for (int i = 0; i < texCount; ++i)
6969
{

‎Packages/com.unity.webrtc/Runtime/Srcipts/MediaStream.cs‎

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,27 @@ namespace Unity.WebRTC
88
{
99
public class MediaStream
1010
{
11-
//to do : c++ create two mediastream named "audio" and "vedio". Actually we only need one.
11+
internal IntPtr nativePtr;
1212
protected List<MediaStreamTrack> mediaStreamTrackList = new List<MediaStreamTrack>();
1313

14-
public MediaStream() : base()
14+
public MediaStream(string label) : base()
1515
{
16-
16+
nativePtr = WebRTC.Context.CreateMediaStream(label);
1717
}
1818

19-
public MediaStream(MediaStreamTrack[] tracks) : base()
19+
public MediaStream(string label, MediaStreamTrack[] tracks) : base()
2020
{
21+
nativePtr = WebRTC.Context.CreateMediaStream(label);
22+
2123
foreach (var t in tracks)
2224
{
23-
mediaStreamTrackList.Add(t);
25+
AddTrack(t);
2426
}
2527
}
2628

2729
public void AddTrack(MediaStreamTrack track)
2830
{
31+
NativeMethods.MediaStreamAddTrack(nativePtr, track.nativePtr);
2932
mediaStreamTrackList.Add(track);
3033
}
3134

@@ -128,16 +131,6 @@ public static class Audio
128131
{
129132
private static bool started = false;
130133
private static AudioInput audioInput = new AudioInput();
131-
public static MediaStream CaptureStream()
132-
{
133-
audioInput.BeginRecording();
134-
started = true;
135-
136-
MediaStream mediaStream = new MediaStream();
137-
AudioStreamTrack audioStreamTrack = new AudioStreamTrack();
138-
mediaStream.AddTrack(audioStreamTrack);
139-
return mediaStream;
140-
}
141134

142135
public static void Update()
143136
{

‎Packages/com.unity.webrtc/Runtime/Srcipts/MediaStreamTrack.cs‎

Lines changed: 2 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,6 @@ public class MediaStreamTrack
99
internal IntPtr nativePtr;
1010
protected string id;
1111
protected TrackKind kind;
12-
internal MediaStreamTrack()
13-
{
14-
15-
}
1612

1713
internal MediaStreamTrack(IntPtr ptr)
1814
{
@@ -40,41 +36,15 @@ private set { }
4036

4137
public class VideoStreamTrack : MediaStreamTrack
4238
{
43-
public VideoStreamTrack(RenderTexture rt) : base()
39+
public VideoStreamTrack(RenderTexture rt) : base(WebRTC.Context.CreateVideoTrack("videoTrack", rt.GetNativeTexturePtr(), rt.width, rt.height))
4440
{
45-
IntPtr nativeVideoStreamPtr = WebRTC.Context.CaptureVideoStream(rt.GetNativeTexturePtr(), rt.width, rt.height);
46-
string nativeVideoStreamID = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(nativeVideoStreamPtr));
47-
48-
int trackSize = 0;
49-
IntPtr tracksNativePtr = NativeMethods.MediaStreamGetVideoTracks(nativeVideoStreamPtr, ref trackSize);
50-
IntPtr[] tracksPtr = new IntPtr[trackSize];
51-
Marshal.Copy(tracksNativePtr, tracksPtr, 0, trackSize);
52-
Marshal.FreeCoTaskMem(tracksNativePtr);
53-
54-
nativePtr = tracksPtr[0];
55-
kind = NativeMethods.MediaStreamTrackGetKind(nativePtr);
56-
id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(nativePtr));
57-
5841
}
5942
}
6043

6144
public class AudioStreamTrack : MediaStreamTrack
6245
{
63-
public AudioStreamTrack() : base()
46+
public AudioStreamTrack() : base(WebRTC.Context.CreateAudioTrack("audioTrack"))
6447
{
65-
IntPtr nativeAudioStreamPtr = WebRTC.Context.CaptureAudioStream();
66-
string nativeAudioStreamID = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamGetID(nativeAudioStreamPtr));
67-
68-
int trackSize = 0;
69-
IntPtr trackNativePtr = NativeMethods.MediaStreamGetAudioTracks(nativeAudioStreamPtr, ref trackSize);
70-
IntPtr[] tracksPtr = new IntPtr[trackSize];
71-
Marshal.Copy(trackNativePtr, tracksPtr, 0, trackSize);
72-
//TODO: Linux compatibility
73-
Marshal.FreeCoTaskMem(trackNativePtr);
74-
75-
nativePtr = tracksPtr[0];
76-
kind = NativeMethods.MediaStreamTrackGetKind(nativePtr);
77-
id = Marshal.PtrToStringAnsi(NativeMethods.MediaStreamTrackGetID(nativePtr));
7848
}
7949
}
8050

‎Packages/com.unity.webrtc/Runtime/Srcipts/WebRTC.cs‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -362,7 +362,11 @@ internal static class NativeMethods
362362
[DllImport(WebRTC.Lib)]
363363
public static extern IntPtr CaptureVideoStream(IntPtr context, IntPtr rt, int width, int height);
364364
[DllImport(WebRTC.Lib)]
365-
public static extern IntPtr CaptureAudioStream(IntPtr context);
365+
public static extern IntPtr CreateMediaStream(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label);
366+
[DllImport(WebRTC.Lib)]
367+
public static extern IntPtr CreateVideoTrack(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label, IntPtr rt, int width, int height);
368+
[DllImport(WebRTC.Lib)]
369+
public static extern IntPtr CreateAudioTrack(IntPtr context, [MarshalAs(UnmanagedType.LPStr, SizeConst = 256)] string label);
366370
[DllImport(WebRTC.Lib)]
367371
public static extern void MediaStreamAddTrack(IntPtr stream, IntPtr track);
368372
[DllImport(WebRTC.Lib)]
@@ -399,9 +403,9 @@ internal struct Context
399403
public static bool ToBool(Context v) { return v; }
400404
public static Context Create(int uid = 0) { return NativeMethods.ContextCreate(uid); }
401405
public void Destroy(int uid = 0) { NativeMethods.ContextDestroy(uid); self = IntPtr.Zero; }
402-
public IntPtr CaptureVideoStream(IntPtr rt, int width, int height) { return NativeMethods.CaptureVideoStream(self, rt, width, height); }
403-
public IntPtr CaptureAudioStream() { return NativeMethods.CaptureAudioStream(self); }
404-
406+
public IntPtr CreateMediaStream(string label) { return NativeMethods.CreateMediaStream(self, label); }
407+
public IntPtr CreateVideoTrack(string label, IntPtr rt, int width, int height) { return NativeMethods.CreateVideoTrack(self, label, rt, width, height); }
408+
public IntPtr CreateAudioTrack(string label) {return NativeMethods.CreateAudioTrack(self, label);}
405409
}
406410
}
407411

‎Packages/com.unity.webrtc/Samples/Example/MediaStreamSample.cs‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ public class MediaStreamSample : MonoBehaviour
1919

2020
private RTCPeerConnection pc1, pc2;
2121
private List<RTCRtpSender> pc1Senders, pc2Senders;
22-
private Unity.WebRTC.MediaStream audioStream, videoStream;
22+
private Unity.WebRTC.MediaStream mediaStream;
2323
private RTCDataChannel dataChannel, remoteDataChannel;
2424
private Coroutine sdpCheck;
2525
private string msg;
@@ -159,14 +159,11 @@ void Pc2OnIceCandidate(RTCIceCandidate candidate)
159159
}
160160
public void AddTracks()
161161
{
162-
foreach (var track in audioStream.GetTracks())
162+
foreach (var track in mediaStream.GetTracks())
163163
{
164164
pc1Senders.Add (pc1.AddTrack(track));
165165
}
166-
foreach(var track in videoStream.GetTracks())
167-
{
168-
pc1Senders.Add(pc1.AddTrack(track));
169-
}
166+
170167
if(!videoUpdateStarted)
171168
{
172169
StartCoroutine(WebRTC.Update());
@@ -212,16 +209,17 @@ void Call()
212209

213210
RTCDataChannelInit conf = new RTCDataChannelInit(true);
214211
dataChannel = pc1.CreateDataChannel("data", ref conf);
215-
audioStream = Audio.CaptureStream();
216212

217213
cam.CreateRenderStreamTexture(1280, 720);
218-
videoStream = new MediaStream();
214+
mediaStream = new MediaStream("mediaStream");
219215
int texCount = cam.GetStreamTextureCount();
220216
for (int i = 0; i < texCount; ++i)
221217
{
222-
videoStream.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i)));
218+
mediaStream.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i)));
223219
}
224220

221+
mediaStream.AddTrack(new AudioStreamTrack());
222+
Audio.Start();
225223

226224
RtImage.texture = cam.targetTexture;
227225

‎Packages/com.unity.webrtc/Tests/Runtime/MediaStreamTest.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,13 @@ public IEnumerator MediaStreamTest_AddAndRemoveMediaStream()
4949

5050

5151
cam.CreateRenderStreamTexture(1280, 720);
52-
MediaStream videoStream = new MediaStream();
52+
MediaStream mediaStream = new MediaStream("mediaStream");
5353
int texCount = cam.GetStreamTextureCount();
5454
for (int i = 0; i < texCount; ++i)
5555
{
56-
pc1Senders.Add(peer1.AddTrack(new VideoStreamTrack(cam.GetStreamTexture(i))));
56+
VideoStreamTrack videoStreamTrack = new VideoStreamTrack(cam.GetStreamTexture(i));
57+
mediaStream.AddTrack(videoStreamTrack);
58+
pc1Senders.Add(peer1.AddTrack(videoStreamTrack));
5759
}
5860

5961
var conf = new RTCDataChannelInit(true);

‎Plugin/WebRTCPlugin/Context.cpp‎

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -307,58 +307,57 @@ namespace WebRTC
307307
nullptr,
308308
nullptr);
309309

310-
mediaStream = peerConnectionFactory->CreateLocalMediaStream("mediaStream");
311310
}
312311

313312
Context::~Context()
314313
{
315314
clients.clear();
316315
peerConnectionFactory = nullptr;
317-
mediaStream = nullptr;
316+
317+
videoTrack = nullptr;
318+
audioTrack = nullptr;
319+
320+
mediaStreamMap.clear();
318321

319322
workerThread->Quit();
320323
workerThread.reset();
321324
signalingThread->Quit();
322325
signalingThread.reset();
323326
}
324327

325-
webrtc::MediaStreamInterface* Context::CreateVideoStream(UnityFrameBuffer* frameBuffer, int32 width, int32 height)
328+
webrtc::MediaStreamInterface* Context::CreateMediaStream(const std::string& stream_id)
326329
{
327-
////TODO: label and stream id should be maintained in some way for multi-stream
328-
auto videoTrack = CreateVideoTrack("video", frameBuffer, width, height);
329-
mediaStream->AddTrack(videoTrack);
330-
return mediaStream.get();
330+
if (mediaStreamMap.count(stream_id) == 0)
331+
{
332+
mediaStreamMap[stream_id] = peerConnectionFactory->CreateLocalMediaStream(stream_id);
333+
}
334+
335+
return mediaStreamMap[stream_id];
331336
}
332337

333-
rtc::scoped_refptr<webrtc::VideoTrackInterface> Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height)
338+
webrtc::MediaStreamTrackInterface* Context::CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height)
334339
{
335340
nvVideoCapturerUnique = std::make_unique<NvVideoCapturer>();
336341
nvVideoCapturer = nvVideoCapturerUnique.get();
337342
nvVideoCapturer->InitializeEncoder(width, height);
338343
pDummyVideoEncoderFactory->SetCapturer(nvVideoCapturer);
339-
auto videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(std::move(nvVideoCapturerUnique)));
344+
345+
videoTrack = peerConnectionFactory->CreateVideoTrack(label, peerConnectionFactory->CreateVideoSource(std::move(nvVideoCapturerUnique)));
340346
nvVideoCapturer->unityRT = frameBuffer;
341347
nvVideoCapturer->StartEncoder();
342348

343349
return videoTrack;
344350
}
345351

346-
webrtc::MediaStreamInterface* Context::CreateAudioStream()
347-
{
348-
auto audioTrack = CreateAudioTrack();
349-
mediaStream->AddTrack(audioTrack);
350-
return mediaStream.get();
351-
}
352-
353-
rtc::scoped_refptr<webrtc::AudioTrackInterface> Context::CreateAudioTrack()
352+
webrtc::MediaStreamTrackInterface* Context::CreateAudioTrack(const std::string& label)
354353
{
355354
//avoid optimization specially for voice
356355
cricket::AudioOptions audioOptions;
357356
audioOptions.auto_gain_control = false;
358357
audioOptions.noise_suppression = false;
359358
audioOptions.highpass_filter = false;
360359
//TODO: label and stream id should be maintained in some way for multi-stream
361-
auto audioTrack = peerConnectionFactory->CreateAudioTrack("audio", peerConnectionFactory->CreateAudioSource(audioOptions));
360+
audioTrack = peerConnectionFactory->CreateAudioTrack(label, peerConnectionFactory->CreateAudioSource(audioOptions));
362361
return audioTrack;
363362
}
364363

‎Plugin/WebRTCPlugin/Context.h‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,9 @@ namespace WebRTC
4242
{
4343
public:
4444
explicit Context(int uid = -1);
45-
webrtc::MediaStreamInterface* CreateVideoStream(UnityFrameBuffer* frameBuffer, int32 width, int32 height);
46-
rtc::scoped_refptr<webrtc::VideoTrackInterface> CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height);
47-
webrtc::MediaStreamInterface* CreateAudioStream();
48-
rtc::scoped_refptr<webrtc::AudioTrackInterface> CreateAudioTrack();
45+
webrtc::MediaStreamInterface* CreateMediaStream(const std::string& stream_id);
46+
webrtc::MediaStreamTrackInterface* CreateVideoTrack(const std::string& label, UnityFrameBuffer* frameBuffer, int32 width, int32 height);
47+
webrtc::MediaStreamTrackInterface* CreateAudioTrack(const std::string& label);
4948
~Context();
5049

5150
PeerConnectionObject* CreatePeerConnection(int id);
@@ -61,11 +60,14 @@ namespace WebRTC
6160
std::map<int, rtc::scoped_refptr<PeerConnectionObject>> clients;
6261
rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> peerConnectionFactory;
6362
DummyVideoEncoderFactory* pDummyVideoEncoderFactory;
64-
rtc::scoped_refptr<webrtc::MediaStreamInterface> mediaStream;
63+
std::map<const std::string, rtc::scoped_refptr<webrtc::MediaStreamInterface>> mediaStreamMap;
6564

6665
NvVideoCapturer* nvVideoCapturer;
6766
std::unique_ptr<NvVideoCapturer> nvVideoCapturerUnique;
6867
rtc::scoped_refptr<DummyAudioDevice> audioDevice;
68+
69+
rtc::scoped_refptr<webrtc::VideoTrackInterface> videoTrack;
70+
rtc::scoped_refptr<webrtc::AudioTrackInterface> audioTrack;
6971
};
7072

7173
class PeerSDPObserver : public webrtc::SetSessionDescriptionObserver

‎Plugin/WebRTCPlugin/WebRTCPlugin.cpp‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,21 @@ namespace WebRTC
2929

3030
extern "C"
3131
{
32-
UNITY_INTERFACE_EXPORT webrtc::MediaStreamInterface* CaptureVideoStream(Context* context, UnityFrameBuffer* rt, int32 width, int32 height)
32+
UNITY_INTERFACE_EXPORT webrtc::MediaStreamInterface* CreateMediaStream(Context* context, const char* label)
3333
{
34-
//context->InitializeEncoder(width, height);
35-
return context->CreateVideoStream(rt, width, height);
34+
return context->CreateMediaStream(label);
3635
}
36+
37+
UNITY_INTERFACE_EXPORT webrtc::MediaStreamTrackInterface* CreateVideoTrack(Context* context, const char* label, UnityFrameBuffer* frameBuffer, int32 width, int32 height)
38+
{
39+
return context->CreateVideoTrack(label, frameBuffer, width, height);
40+
}
41+
42+
UNITY_INTERFACE_EXPORT webrtc::MediaStreamTrackInterface* CreateAudioTrack(Context* context, const char* label)
43+
{
44+
return context->CreateAudioTrack(label);
45+
}
46+
3747
//TODO: Multi-track support
3848
UNITY_INTERFACE_EXPORT void StopMediaStreamTrack(webrtc::MediaStreamTrackInterface* track)
3949
{
@@ -45,11 +55,6 @@ extern "C"
4555
return ContextManager::GetNvEncSupported();
4656
}
4757

48-
UNITY_INTERFACE_EXPORT webrtc::MediaStreamInterface* CaptureAudioStream(Context* context)
49-
{
50-
return context->CreateAudioStream();
51-
}
52-
5358
UNITY_INTERFACE_EXPORT void MediaStreamAddTrack(webrtc::MediaStreamInterface* stream, webrtc::MediaStreamTrackInterface* track)
5459
{
5560
if (track->kind() == "audio")
@@ -61,6 +66,7 @@ extern "C"
6166
stream->AddTrack((webrtc::VideoTrackInterface*)track);
6267
}
6368
}
69+
6470
UNITY_INTERFACE_EXPORT void MediaStreamRemoveTrack(webrtc::MediaStreamInterface* stream, webrtc::MediaStreamTrackInterface* track)
6571
{
6672
if (track->kind() == "audio")

0 commit comments

Comments
 (0)