Skip to content

Commit

Permalink
Improvements in handling VoIP connections
Browse files Browse the repository at this point in the history
  • Loading branch information
@hantu85
hantu85 committed Dec 17, 2018
1 parent 1accf3a commit 4aa0f4d
Show file tree
Hide file tree
Showing 10 changed files with 373 additions and 253 deletions.
10 changes: 4 additions & 6 deletions BeagleIM.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
objects = {

/* Begin PBXBuildFile section */
FE032F5421A1A4D70039FE71 /* WebRTC.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = FE032F5321A1A4D70039FE71 /* WebRTC.framework */; };
FE032F5521A1A4D70039FE71 /* WebRTC.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = FE032F5321A1A4D70039FE71 /* WebRTC.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
FE28097D21652A7C002F5BD0 /* VCardEditorViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = FE28097C21652A7C002F5BD0 /* VCardEditorViewController.swift */; };
FE28097F2166892D002F5BD0 /* AccountDetailsViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = FE28097E2166892D002F5BD0 /* AccountDetailsViewController.swift */; };
FE2809852169FB75002F5BD0 /* Alert.swift in Sources */ = {isa = PBXBuildFile; fileRef = FE2809842169FB74002F5BD0 /* Alert.swift */; };
Expand Down Expand Up @@ -69,6 +67,7 @@
FE8C9F5421B3F6BD00BE8332 /* container-migration.plist in Resources */ = {isa = PBXBuildFile; fileRef = FE8C9F5321B3F6BD00BE8332 /* container-migration.plist */; };
FE8C9F5621B40F9000BE8332 /* RegisterAccountController.swift in Sources */ = {isa = PBXBuildFile; fileRef = FE8C9F5521B40F9000BE8332 /* RegisterAccountController.swift */; };
FE9486B221BBD7A600C8DE49 /* NSButtonNoSizeWhenHidden.swift in Sources */ = {isa = PBXBuildFile; fileRef = FE9486B121BBD7A600C8DE49 /* NSButtonNoSizeWhenHidden.swift */; };
FEA77CD221C7C47000E955A8 /* WebRTC.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = FEA77CD121C7C47000E955A8 /* WebRTC.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
FEA8941F2156625E004C3B51 /* ConfigureRoomViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = FEA8941E2156625E004C3B51 /* ConfigureRoomViewController.swift */; };
FEB4EAFC2162A3DC00F8A9E2 /* AutoresizineTextView.swift in Sources */ = {isa = PBXBuildFile; fileRef = FEB4EAFB2162A3DC00F8A9E2 /* AutoresizineTextView.swift */; };
FEB4EAFE2163714300F8A9E2 /* RoundedScrollView.swift in Sources */ = {isa = PBXBuildFile; fileRef = FEB4EAFD2163714300F8A9E2 /* RoundedScrollView.swift */; };
Expand Down Expand Up @@ -124,16 +123,15 @@
dstPath = "";
dstSubfolderSpec = 10;
files = (
FEA77CD221C7C47000E955A8 /* WebRTC.framework in Embed Frameworks */,
FE4E8DE12160E7AA0010D88E /* TigaseSwift.framework in Embed Frameworks */,
FE032F5521A1A4D70039FE71 /* WebRTC.framework in Embed Frameworks */,
);
name = "Embed Frameworks";
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXCopyFilesBuildPhase section */

/* Begin PBXFileReference section */
FE032F5321A1A4D70039FE71 /* WebRTC.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = WebRTC.framework; sourceTree = "<group>"; };
FE28097C21652A7C002F5BD0 /* VCardEditorViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VCardEditorViewController.swift; sourceTree = "<group>"; };
FE28097E2166892D002F5BD0 /* AccountDetailsViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccountDetailsViewController.swift; sourceTree = "<group>"; };
FE2809842169FB74002F5BD0 /* Alert.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Alert.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -194,6 +192,7 @@
FE8C9F5521B40F9000BE8332 /* RegisterAccountController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RegisterAccountController.swift; sourceTree = "<group>"; };
FE9486B121BBD7A600C8DE49 /* NSButtonNoSizeWhenHidden.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NSButtonNoSizeWhenHidden.swift; sourceTree = "<group>"; };
FE9D88B3217652DF006CF0D9 /* BeagleIM-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "BeagleIM-Bridging-Header.h"; sourceTree = "<group>"; };
FEA77CD121C7C47000E955A8 /* WebRTC.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = WebRTC.framework; sourceTree = "<group>"; };
FEA8941E2156625E004C3B51 /* ConfigureRoomViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConfigureRoomViewController.swift; sourceTree = "<group>"; };
FEB4EAFB2162A3DC00F8A9E2 /* AutoresizineTextView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AutoresizineTextView.swift; sourceTree = "<group>"; };
FEB4EAFD2163714300F8A9E2 /* RoundedScrollView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RoundedScrollView.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -241,7 +240,6 @@
FE83E365217B0F89001DEF09 /* AVFoundation.framework in Frameworks */,
FEF9D12B2082804E00B09B6C /* TigaseSwift.framework in Frameworks */,
FEF9CE2220827F0000B09B6C /* libsqlite3.0.tbd in Frameworks */,
FE032F5421A1A4D70039FE71 /* WebRTC.framework in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -395,7 +393,7 @@
FED2ECE72066B01C00F506A5 = {
isa = PBXGroup;
children = (
FE032F5321A1A4D70039FE71 /* WebRTC.framework */,
FEA77CD121C7C47000E955A8 /* WebRTC.framework */,
FED2ECF22066B01C00F506A5 /* BeagleIM */,
FED2ED052066B01C00F506A5 /* BeagleIMTests */,
FED2ED102066B01C00F506A5 /* BeagleIMUITests */,
Expand Down
3 changes: 3 additions & 0 deletions BeagleIM/AppDelegate.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
//

import Cocoa
import WebRTC
import TigaseSwift
import UserNotifications
import AVFoundation
Expand Down Expand Up @@ -69,6 +70,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSUserNotificationCenterDele

func applicationDidFinishLaunching(_ aNotification: Notification) {
Settings.initialize();
RTCInitializeSSL();

if #available(macOS 10.14, *) {
updateAppearance();
Expand Down Expand Up @@ -165,6 +167,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSUserNotificationCenterDele
func applicationWillTerminate(_ aNotification: Notification) {
// Insert code here to tear down your application
XmppService.instance.disconnectClients();
RTCCleanupSSL();
}

func applicationShouldHandleReopen(_ sender: NSApplication, hasVisibleWindows flag: Bool) -> Bool {
Expand Down
168 changes: 112 additions & 56 deletions BeagleIM/service/JingleManager.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ class JingleManager: XmppServiceEventHandler {
// }

func sessionInitiated(event e: JingleModule.JingleEvent) {

guard let content = e.contents.first, let description = content.description as? Jingle.RTP.Description else {
return;
}
Expand All @@ -215,24 +216,37 @@ class JingleManager: XmppServiceEventHandler {
func sessionTerminated(event e: JingleModule.JingleEvent) {
guard let session = session(for: e.sessionObject.userBareJid!, with: e.jid, sid: e.sid) else {
return;
}
}
session.terminate();
}

func transportInfo(event e: JingleModule.JingleEvent) {
print("processing transport info");

// add support for multiple contents...
guard let content = e.contents.first, let transport = content.transports.first as? Jingle.Transport.ICEUDPTransport else {
return;
}

guard let session = self.session(for: e.sessionObject.userBareJid!, with: e.jid, sid: e.sid) else {
return;
}

transport.candidates.forEach { (candidate) in
session.addCandidate(candidate, for: content.name);
e.contents.forEach { (content) in
content.transports.forEach({ (trans) in
if let transport = trans as? Jingle.Transport.ICEUDPTransport {
transport.candidates.forEach({ (candidate) in
session.addCandidate(candidate, for: content.name);
})
}
})
}
// // add support for multiple contents...
// guard let content = e.contents.first, let transport = content.transports.first as? Jingle.Transport.ICEUDPTransport else {
// return;
// }
//
// guard let session = self.session(for: e.sessionObject.userBareJid!, with: e.jid, sid: e.sid) else {
// return;
// }
//
// transport.candidates.forEach { (candidate) in
// session.addCandidate(candidate, for: content.name);
// }
}

// func createLocalStream(audio: Bool, video: Bool) -> RTCMediaStream {
Expand Down Expand Up @@ -295,7 +309,8 @@ class JingleManager: XmppServiceEventHandler {
fileprivate(set) var sid: String;
let role: Jingle.Content.Creator;

var remoteCandidates: [[String]] = [];
var remoteCandidates: [[String]]? = [];
var localCandidates: [RTCIceCandidate]? = [];

init(account: BareJID, jid: JID, sid: String? = nil, role: Jingle.Content.Creator) {
self.account = account;
Expand Down Expand Up @@ -343,20 +358,7 @@ class JingleManager: XmppServiceEventHandler {

func accepted(sdpAnswer: SDP) {
self.state = .connecting;
delegate?.sessionAccepted(session: self);

print("setting remote description:", sdpAnswer.toString());
let sessDesc = RTCSessionDescription(type: .answer, sdp: sdpAnswer.toString());
peerConnection?.setRemoteDescription(sessDesc, completionHandler: { (error) in
guard error == nil else {
// for now we are closing but maybe we should send content-remove or content-modify instead....
print("sdp offset:", self.peerConnection?.localDescription!.sdp);
print("failed to set sdp answer:", sdpAnswer.toString());
_ = self.terminate();
return;
}
self.remoteDescriptionSet();
});
delegate?.sessionAccepted(session: self, sdpAnswer: sdpAnswer);
}

func decline() -> Bool {
Expand All @@ -366,6 +368,10 @@ class JingleManager: XmppServiceEventHandler {
}

jingleModule.declineSession(with: jid, sid: sid);

self.delegate?.sessionTerminated(session: self);
peerConnection?.close();
peerConnection = nil;
return true;
}

Expand All @@ -386,6 +392,7 @@ class JingleManager: XmppServiceEventHandler {
jingleModule.terminateSession(with: jid, sid: sid);

self.delegate?.sessionTerminated(session: self);
peerConnection?.close();
peerConnection = nil;

JingleManager.instance.close(session: self);
Expand All @@ -395,18 +402,21 @@ class JingleManager: XmppServiceEventHandler {
func addCandidate(_ candidate: Jingle.Transport.ICEUDPTransport.Candidate, for contentName: String) {
let sdp = candidate.toSDP();

guard let remoteDesc = peerConnection?.remoteDescription else {
self.remoteCandidates.append([contentName, sdp]);
return;
if remoteCandidates != nil {
remoteCandidates?.append([contentName, sdp]);
} else {
self.addCandidate(sdp: sdp, for: contentName);
}
self.addCandidate(sdp: sdp, for: contentName);
}

func remoteDescriptionSet() {
let tmp = remoteCandidates;
remoteCandidates.removeAll();
tmp.forEach { (arr) in
self.addCandidate(sdp: arr[1], for: arr[0]);
JingleManager.instance.dispatcher.async {
if let tmp = self.remoteCandidates {
self.remoteCandidates = nil;
tmp.forEach { (arr) in
self.addCandidate(sdp: arr[1], for: arr[0]);
}
}
}
}

Expand All @@ -415,34 +425,48 @@ class JingleManager: XmppServiceEventHandler {
}

fileprivate func addCandidate(sdp: String, for contentName: String) {
guard let lines = peerConnection?.remoteDescription?.sdp.split(separator: "\r\n").map({ (s) -> String in
return String(s);
}) else {
return;
}

guard let midIdx = lines.firstIndex(of: "a=mid:\(contentName)") else {
return;
}

let sublines = lines[0...midIdx];

guard let idx = sublines.lastIndex(where: { (s) -> Bool in
return s.starts(with: "m=");
}) else {
return;
DispatchQueue.main.async {
guard let lines = self.peerConnection?.remoteDescription?.sdp.split(separator: "\r\n").map({ (s) -> String in
return String(s);
}) else {
return;
}

let contents = lines.filter { (line) -> Bool in
return line.starts(with: "a=mid:");
};

let idx = contents.firstIndex(of: "a=mid:\(contentName)") ?? lines.filter({ (line) -> Bool in
return line.starts(with: "m=")
}).firstIndex(where: { (line) -> Bool in
return line.starts(with: "m=\(contentName) ");
}) ?? 0;

// guard let midIdx = lines.firstIndex(of: "a=mid:\(contentName)") else {
// return;
// }
//
// let sublines = lines[0...midIdx];
//
// guard let idx = sublines.lastIndex(where: { (s) -> Bool in
// return s.starts(with: "m=");
// }) else {
// return;
// }
// if peer connection is nil we should queue those candidates...
print("adding candidate for:", idx, "name:", contentName, "sdp:", sdp)
self.peerConnection?.add(RTCIceCandidate(sdp: sdp, sdpMLineIndex: Int32(idx), sdpMid: contentName));
}
// if peer connection is nil we should queue those candidates...
peerConnection?.add(RTCIceCandidate(sdp: sdp, sdpMLineIndex: Int32(idx), sdpMid: contentName));
}

func peerConnection(_ peerConnection: RTCPeerConnection, didChange stateChanged: RTCSignalingState) {
print("signaling state:", stateChanged.rawValue);
}

func peerConnection(_ peerConnection: RTCPeerConnection, didAdd stream: RTCMediaStream) {
if stream.videoTracks.count > 0 {
self.delegate?.didAdd(remoteVideoTrack: stream.videoTracks[0]);
}
// if stream.videoTracks.count > 0 {
// self.delegate?.didAdd(remoteVideoTrack: stream.videoTracks[0]);
// }
}

func peerConnection(_ peerConnection: RTCPeerConnection, didRemove stream: RTCMediaStream) {
Expand Down Expand Up @@ -472,6 +496,7 @@ class JingleManager: XmppServiceEventHandler {
}

func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceConnectionState) {
print("ice connection state:", newState.rawValue);
if newState == .connected {
self.state = .connected;
} else if (state == .connected && (newState == .disconnected || newState == .failed)) {
Expand All @@ -484,7 +509,36 @@ class JingleManager: XmppServiceEventHandler {
}

func peerConnection(_ peerConnection: RTCPeerConnection, didGenerate candidate: RTCIceCandidate) {
guard let jingleCandidate = Jingle.Transport.ICEUDPTransport.Candidate(fromSDP: candidate.sdp) else {
print("generated candidate for:", candidate.sdpMid, ", index:", candidate.sdpMLineIndex, "full SDP:", (peerConnection.localDescription?.sdp ?? ""));

JingleManager.instance.dispatcher.async {
if self.localCandidates == nil {
self.sendLocalCandidate(candidate);
} else {
self.localCandidates?.append(candidate);
}
}

// if (!transportInfo(contentName: mid, creator: role, transport: Jingle.Transport.ICEUDPTransport(pwd: transport.pwd, ufrag: transport.ufrag, candidates: [jingleCandidate]))) {
// self.onError(.remote_server_timeout);
// }
}

func localDescriptionSet() {
JingleManager.instance.dispatcher.async {
if let tmp = self.localCandidates {
self.localCandidates = nil;
tmp.forEach({ (candidate) in
self.sendLocalCandidate(candidate);
})
}
}
}

fileprivate func sendLocalCandidate(_ candidate: RTCIceCandidate) {
print("sending candidate for:", candidate.sdpMid, ", index:", candidate.sdpMLineIndex, "full SDP:", (self.peerConnection?.localDescription?.sdp ?? ""));

guard let jingleCandidate = Jingle.Transport.ICEUDPTransport.Candidate(fromSDP: candidate.sdp), let peerConnection = self.peerConnection else {
return;
}
guard let mid = candidate.sdpMid else {
Expand All @@ -503,8 +557,10 @@ class JingleManager: XmppServiceEventHandler {
return;
}

if (!transportInfo(contentName: mid, creator: role, transport: Jingle.Transport.ICEUDPTransport(pwd: transport.pwd, ufrag: transport.ufrag, candidates: [jingleCandidate]))) {
self.onError(.remote_server_timeout);
DispatchQueue.main.asyncAfter(deadline: DispatchTime.now() + 0.1) {
if (!self.transportInfo(contentName: mid, creator: self.role, transport: Jingle.Transport.ICEUDPTransport(pwd: transport.pwd, ufrag: transport.ufrag, candidates: [jingleCandidate]))) {
self.onError(.remote_server_timeout);
}
}
}

Expand Down
Loading

0 comments on commit 4aa0f4d

Please sign in to comment.