Skip to content

Commit

Permalink
Update WebRTC library and accomodate API changes.
Browse files Browse the repository at this point in the history
  • Loading branch information
@bitbound
bitbound committed Aug 7, 2020
1 parent ab09c81 commit ca18f3c
Show file tree
Hide file tree
Showing 9 changed files with 107 additions and 47 deletions.
2 changes: 1 addition & 1 deletion Desktop.Core/Desktop.Core.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="3.1.6" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="3.1.6" />
<PackageReference Include="Microsoft.Extensions.Logging.EventLog" Version="3.1.6" />
<PackageReference Include="Microsoft.MixedReality.WebRTC" Version="1.0.3" />
<PackageReference Include="Microsoft.MixedReality.WebRTC" Version="2.0.0" />
<PackageReference Include="System.Drawing.Common" Version="4.7.0" />
<PackageReference Include="System.IO.FileSystem.AccessControl" Version="4.7.0" />
</ItemGroup>
Expand Down
6 changes: 3 additions & 3 deletions Desktop.Core/Models/Viewer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,11 @@ public async Task InitializeWebRtc()
RtcSession = WebRtcSessionFactory.GetNewSession(this);
RtcSession.LocalSdpReady += async (sender, sdp) =>
{
await CasterSocket.SendRtcOfferToBrowser(sdp, ViewerConnectionID, iceServers);
await CasterSocket.SendRtcOfferToBrowser(sdp.Content, ViewerConnectionID, iceServers);
};
RtcSession.IceCandidateReady += async (sender, args) =>
RtcSession.IceCandidateReady += async (sender, candidate) =>
{
await CasterSocket.SendIceCandidateToBrowser(args.candidate, args.sdpMlineIndex, args.sdpMid, ViewerConnectionID);
await CasterSocket.SendIceCandidateToBrowser(candidate.Content, candidate.SdpMlineIndex, candidate.SdpMid, ViewerConnectionID);
};

await RtcSession.Init(iceServers);
Expand Down
121 changes: 93 additions & 28 deletions Desktop.Core/Services/WebRtcSession.cs
View file
Original file line number Diff line number Diff line change
@@ -1,42 +1,53 @@
using MessagePack;
using Microsoft.MixedReality.WebRTC;
using Remotely.Desktop.Core.Models;
using Remotely.Shared.Models;
using Remotely.Shared.Models.RtcDtos;
using Remotely.Shared.Utilities;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Threading.Tasks;

namespace Remotely.Desktop.Core.Services
{
public class WebRtcSession : IDisposable
{
public WebRtcSession(IRtcMessageHandler rtcMessageHandler)
public WebRtcSession(Viewer viewer, IRtcMessageHandler rtcMessageHandler)
{
Viewer = viewer;
RtcMessageHandler = rtcMessageHandler;
}

public event EventHandler<(string candidate, int sdpMlineIndex, string sdpMid)> IceCandidateReady;
public event EventHandler<IceCandidate> IceCandidateReady;

public event EventHandler<string> LocalSdpReady;
public event EventHandler<SdpMessage> LocalSdpReady;

public ulong CurrentBuffer { get; private set; }
public bool IsDataChannelOpen => CaptureChannel?.State == DataChannel.ChannelState.Open;
public bool IsPeerConnected => PeerConnection?.IsConnected == true;
public bool IsPeerConnected => PeerSession?.IsConnected == true;
private DataChannel CaptureChannel { get; set; }
private IceServerModel[] IceServers { get; set; }
private PeerConnection PeerConnection { get; set; }
private PeerConnection PeerSession { get; set; }
private IRtcMessageHandler RtcMessageHandler { get; }
private Transceiver Transceiver { get; set; }
private ExternalVideoTrackSource VideoSource { get; set; }
private Viewer Viewer { get; }

public void AddIceCandidate(string sdpMid, int sdpMlineIndex, string candidate)
{
PeerConnection.AddIceCandidate(sdpMid, sdpMlineIndex, candidate);
PeerSession.AddIceCandidate(new IceCandidate()
{
Content = candidate,
SdpMid = sdpMid,
SdpMlineIndex = sdpMlineIndex
});
}

public void Dispose()
{
CaptureChannel?.Dispose();
PeerConnection?.Dispose();
PeerSession?.Dispose();
}

public async Task Init(IceServerModel[] iceServers)
Expand All @@ -45,7 +56,7 @@ public async Task Init(IceServerModel[] iceServers)

IceServers = iceServers;

PeerConnection = new PeerConnection();
PeerSession = new PeerConnection();

var iceList = IceServers.Select(x => new IceServer()
{
Expand All @@ -59,18 +70,28 @@ public async Task Init(IceServerModel[] iceServers)
IceServers = iceList
};

await PeerConnection.InitializeAsync(config);
await PeerSession.InitializeAsync(config);

PeerSession.LocalSdpReadytoSend += PeerSession_LocalSdpReadytoSend; ;
PeerSession.Connected += PeerConnection_Connected;
PeerSession.IceStateChanged += PeerConnection_IceStateChanged;
PeerSession.IceCandidateReadytoSend += PeerSession_IceCandidateReadytoSend; ;

PeerConnection.LocalSdpReadytoSend += PeerConnection_LocalSdpReadytoSend;
PeerConnection.Connected += PeerConnection_Connected;
PeerConnection.IceStateChanged += PeerConnection_IceStateChanged;
PeerConnection.IceCandidateReadytoSend += PeerConnection_IceCandidateReadytoSend;
CaptureChannel = await PeerConnection.AddDataChannelAsync("ScreenCapture", true, true);
CaptureChannel = await PeerSession.AddDataChannelAsync("ScreenCapture", true, true);
CaptureChannel.BufferingChanged += DataChannel_BufferingChanged;
CaptureChannel.MessageReceived += CaptureChannel_MessageReceived;
CaptureChannel.StateChanged += CaptureChannel_StateChanged;
PeerConnection.CreateOffer();

//VideoSource = ExternalVideoTrackSource.CreateFromArgb32Callback(GetCaptureFrame);
//Transceiver = PeerSession.AddTransceiver(MediaKind.Video);
//Transceiver.LocalVideoTrack = LocalVideoTrack.CreateFromSource(VideoSource, new LocalVideoTrackInitConfig()
//{
// trackName = "ScreenCapture"
//});

PeerSession.CreateOffer();
}

public void SendAudioSample(byte[] audioSample)
{
SendDto(new AudioSampleDto(audioSample));
Expand Down Expand Up @@ -116,6 +137,7 @@ public void SendMachineName(string machineName)
{
SendDto(new MachineNameDto(machineName));
}

public void SendScreenData(string selectedScreen, string[] displayNames)
{
SendDto(new ScreenDataDto(selectedScreen, displayNames));
Expand All @@ -125,19 +147,32 @@ public void SendScreenSize(int width, int height)
{
SendDto(new ScreenSizeDto(width, height));
}

public void SendWindowsSessions(List<WindowsSession> windowsSessions)
{
SendDto(new WindowsSessionsDto(windowsSessions));
}

public void SetRemoteDescription(string type, string sdp)
public async Task SetRemoteDescription(string type, string sdp)
{
PeerConnection.SetRemoteDescription(type, sdp);
if (type == "offer")
if (!Enum.TryParse<SdpMessageType>(type, true, out var sdpMessageType))
{
PeerConnection.CreateAnswer();
Logger.Write("Unable to parse remote WebRTC description type.");
return;
}

await PeerSession.SetRemoteDescriptionAsync(new SdpMessage()
{
Content = sdp,
Type = sdpMessageType
});

if (sdpMessageType == SdpMessageType.Offer)
{
PeerSession.CreateAnswer();
}
}

private async void CaptureChannel_MessageReceived(byte[] obj)
{
Logger.Debug($"DataChannel message received. Size: {obj.Length}");
Expand All @@ -152,31 +187,61 @@ private async void CaptureChannel_StateChanged()
await Init(IceServers);
}
}

private void DataChannel_BufferingChanged(ulong previous, ulong current, ulong limit)
{
CurrentBuffer = current;
}

//private void GetCaptureFrame(in FrameRequest request)
//{
// Viewer.Capturer.GetNextFrame();
// using (var bitmapCopy = (Bitmap)Viewer.Capturer.CurrentFrame.Clone())
// {
// var bitmapData = bitmapCopy.LockBits(
// Viewer.Capturer.CurrentScreenBounds,
// System.Drawing.Imaging.ImageLockMode.ReadOnly,
// System.Drawing.Imaging.PixelFormat.Format32bppArgb);

// try
// {
// var frame = new Argb32VideoFrame()
// {
// data = bitmapData.Scan0,
// height = (uint)bitmapCopy.Height,
// width = (uint)bitmapCopy.Width,
// stride = bitmapData.Stride
// };
// request.CompleteRequest(in frame);
// }
// finally
// {
// bitmapCopy.UnlockBits(bitmapData);
// }

// }

//}

private void PeerConnection_Connected()
{
Logger.Debug("PeerConnection connected.");
}

private void PeerConnection_IceCandidateReadytoSend(string candidate, int sdpMlineindex, string sdpMid)
{
Logger.Debug("Ice candidate ready to send.");
IceCandidateReady?.Invoke(this, (candidate, sdpMlineindex, sdpMid));
}

private void PeerConnection_IceStateChanged(IceConnectionState newState)
{
Logger.Debug($"Ice state changed to {newState}.");
}

private void PeerConnection_LocalSdpReadytoSend(string type, string sdp)
private void PeerSession_IceCandidateReadytoSend(IceCandidate candidate)
{
Logger.Debug("Ice candidate ready to send.");
IceCandidateReady?.Invoke(this, candidate);
}
private void PeerSession_LocalSdpReadytoSend(SdpMessage message)
{
Logger.Debug($"Local SDP ready.");
LocalSdpReady?.Invoke(this, sdp);
LocalSdpReady?.Invoke(this, message);
}
private void SendDto<T>(T dto)
{
Expand Down
2 changes: 1 addition & 1 deletion Desktop.Core/Services/WebRtcSessionFactory.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public WebRtcSession GetNewSession(Viewer viewer)
ClipboardService,
FileDownloadService);

return new WebRtcSession(messageHandler);
return new WebRtcSession(viewer, messageHandler);
}
}
}
6 changes: 3 additions & 3 deletions Desktop.Linux/Desktop.Linux.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@
<None Remove="Assets\Remotely_Icon.png" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Avalonia" Version="0.9.11" />
<PackageReference Include="Avalonia.Desktop" Version="0.9.11" />
<PackageReference Include="Avalonia.ReactiveUI" Version="0.9.11" />
<PackageReference Include="Avalonia" Version="0.9.12" />
<PackageReference Include="Avalonia.Desktop" Version="0.9.12" />
<PackageReference Include="Avalonia.ReactiveUI" Version="0.9.12" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Desktop.Core\Desktop.Core.csproj" />
Expand Down
2 changes: 1 addition & 1 deletion Desktop.Win/Desktop.Win.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="Microsoft.MixedReality.WebRTC" Version="1.0.3" />
<PackageReference Include="Microsoft.MixedReality.WebRTC" Version="2.0.0" />
<PackageReference Include="NAudio" Version="1.10.0" />
<PackageReference Include="SharpDX" Version="4.2.0" />
<PackageReference Include="SharpDX.Direct3D11" Version="4.2.0" />
Expand Down
11 changes: 3 additions & 8 deletions Desktop.Win/Services/ScreenCapturerWin.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,19 +74,14 @@ public void GetNextFrame()
{
try
{
Win32Interop.SwitchToInputDesktop();

if (NeedsInit)
{
Logger.Write("Init needed in DXCapture. Switching desktops.");
if (Win32Interop.SwitchToInputDesktop())
{
Win32Interop.GetCurrentDesktop(out var desktopName);
Logger.Write($"Switch to desktop {desktopName} after capture error in DXCapture.");
}
Logger.Write("Init needed in GetNextFrame.");
Init();
}

Win32Interop.SwitchToInputDesktop();

PreviousFrame?.Dispose();
PreviousFrame = (Bitmap)CurrentFrame.Clone();

Expand Down
2 changes: 1 addition & 1 deletion Server/Server.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="3.1.6" />
<PackageReference Include="Microsoft.Extensions.Logging.EventLog" Version="3.1.6" />
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="3.1.3" PrivateAssets="All" />
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="3.1.4" PrivateAssets="All" />
<PackageReference Include="NETStandard.Library" Version="2.0.3" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="3.1.4" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="5.5.1" />
Expand Down
2 changes: 1 addition & 1 deletion Tests/Tests.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="3.1.6" />
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="3.1.6" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.0" />
<PackageReference Include="MSTest.TestAdapter" Version="2.1.2" />
<PackageReference Include="MSTest.TestFramework" Version="2.1.2" />
<PackageReference Include="coverlet.collector" Version="1.3.0">
Expand Down

0 comments on commit ca18f3c

Please sign in to comment.