Skip to content

Commit

Permalink
fix bind media and sdp
Browse files Browse the repository at this point in the history
  • Loading branch information
@shinyoshiaki
shinyoshiaki committed May 29, 2022
1 parent df36807 commit 32a3610
Show file tree
Hide file tree
Showing 4 changed files with 118 additions and 63 deletions.
162 changes: 103 additions & 59 deletions packages/webrtc/src/peerConnection.ts
View file
Expand Up @@ -230,11 +230,19 @@ export class RTCPeerConnection extends EventTarget {

async createOffer() {
await this.ensureCerts();
const description = this.buildOfferSdp();
return description.toJSON();
}

buildOfferSdp() {
this.transceivers.forEach((transceiver) => {
transceiver.codecs = this.configuration.codecs[transceiver.kind];
transceiver.headerExtensions =
this.configuration.headerExtensions[transceiver.kind];
if (transceiver.codecs.length === 0) {
transceiver.codecs = this.configuration.codecs[transceiver.kind];
}
if (transceiver.headerExtensions.length === 0) {
transceiver.headerExtensions =
this.configuration.headerExtensions[transceiver.kind];
}
});

const description = new SessionDescription();
Expand All @@ -253,22 +261,24 @@ export class RTCPeerConnection extends EventTarget {
return;
}
if (m.kind === "application") {
if (!this.sctpTransport) {
throw new Error("sctpTransport not found");
}
this.sctpTransport.mLineIndex = i;
description.media.push(
createMediaDescriptionForSctp(this.sctpTransport!, mid)
createMediaDescriptionForSctp(this.sctpTransport)
);
} else {
const transceiver = this.getTransceiverByMid(mid);
if (!transceiver) {
log("transceiver by mid not found", mid);
return;
throw new Error("transceiver not found");
}
transceiver.mLineIndex = i;
description.media.push(
createMediaDescriptionForTransceiver(
transceiver,
this.cname,
transceiver.direction,
mid
transceiver.direction
)
);
}
Expand All @@ -279,26 +289,27 @@ export class RTCPeerConnection extends EventTarget {
.filter((t) => !description.media.find((m) => m.rtp.muxId === t.mid))
.forEach((transceiver) => {
transceiver.mLineIndex = description.media.length;
if (transceiver.mid == undefined) {
transceiver.mid = allocateMid(this.seenMid) + "_srtp";

This comment has been minimized.

Sign in to view

Copy link
@koush

koush Jul 1, 2022

Collaborator

This line causes issues with google cam gen 2.

This comment has been minimized.

Sign in to view

Copy link
@koush

koush Jul 1, 2022

Collaborator

referenced issue #230
}
description.media.push(
createMediaDescriptionForTransceiver(
transceiver,
this.cname,
transceiver.direction,
allocateMid(this.seenMid)
transceiver.direction
)
);
});

if (
this.sctpTransport &&
!description.media.find((m) => this.sctpTransport!.mid === m.rtp.muxId)
!description.media.find((m) => m.kind === "application")
) {
description.media.push(
createMediaDescriptionForSctp(
this.sctpTransport,
allocateMid(this.seenMid)
)
);
this.sctpTransport.mLineIndex = description.media.length;
if (this.sctpTransport.mid == undefined) {
this.sctpTransport.mid = allocateMid(this.seenMid) + "_sctp";
}
description.media.push(createMediaDescriptionForSctp(this.sctpTransport));
}

if (this.configuration.bundlePolicy !== "disable") {
Expand All @@ -309,7 +320,7 @@ export class RTCPeerConnection extends EventTarget {
description.group.push(bundle);
}

return description.toJSON();
return description;
}

createDataChannel(
Expand Down Expand Up @@ -392,7 +403,7 @@ export class RTCPeerConnection extends EventTarget {
});
};

private createTransport(srtpProfiles: Profile[] = [], mLineIndex = 0) {
private createTransport(srtpProfiles: Profile[] = []) {
const [existing] = this.iceTransports;

// Gather ICE candidates for only one track. If the remote endpoint is not bundle-aware, negotiate only one media track.
Expand Down Expand Up @@ -423,14 +434,19 @@ export class RTCPeerConnection extends EventTarget {

iceTransport.iceGather.onIceCandidate = (candidate) => {
if (!this.localDescription) return;
const sdp = SessionDescription.parse(this.localDescription.sdp);
const media = sdp.media[mLineIndex];
if (!media) {
log("media not exist");
return;
const transceiver = this.transceivers.find(

This comment has been minimized.

Sign in to view

Copy link
@koush

koush Jun 7, 2022 •
edited

Collaborator

This change broke candidate trickling. It only sends candidates from mlineIndex=1 in cases where that is the first one in the list when a=group:BUNDLE 0 1. (transceivers are added in different order in offer vs answer sdp)

fix:

const transceivers = this.transceivers
  .filter((t) => t.dtlsTransport.iceTransport.id === iceTransport.id)
  .sort((t1, t2) => t1.mLineIndex! - t2.mLineIndex!);
const [transceiver] = transceivers;

I'm not entirely sure this is correct though. But it's on the right path. When bundling, it should send 0 for the mlineindex if 0 and 1 are bundled, I think. It resolved my issue.
(t) => t.dtlsTransport.iceTransport.id === iceTransport.id
);
if (transceiver) {
candidate.sdpMLineIndex = transceiver.mLineIndex;
candidate.sdpMid = transceiver.mid;
}
if (
this.sctpTransport?.dtlsTransport.iceTransport.id === iceTransport.id
) {
candidate.sdpMLineIndex = this.sctpTransport.mLineIndex;
candidate.sdpMid = this.sctpTransport.mid;
}
candidate.sdpMLineIndex = mLineIndex;
candidate.sdpMid = media.rtp.muxId;
candidate.foundation = "candidate:" + candidate.foundation;

this.onIceCandidate.execute(candidate.toJSON());
Expand Down Expand Up @@ -541,9 +557,10 @@ export class RTCPeerConnection extends EventTarget {
this.setLocal(description);

// # gather candidates
for (const iceTransport of this.iceTransports) {
await iceTransport.iceGather.gather();
}
await Promise.all(
this.iceTransports.map((iceTransport) => iceTransport.iceGather.gather())
);

description.media
.filter((m) => ["audio", "video"].includes(m.kind))
.forEach((m, i) => {
Expand Down Expand Up @@ -581,20 +598,54 @@ export class RTCPeerConnection extends EventTarget {
}
}

private getTransportByMid(mid: string) {
let iceTransport: RTCIceTransport | undefined;

const transceiver = this.transceivers.find((t) => t.mid === mid);
if (transceiver) {
iceTransport = transceiver.dtlsTransport.iceTransport;
} else if (!iceTransport && this.sctpTransport?.mid === mid) {
iceTransport = this.sctpTransport?.dtlsTransport.iceTransport;
}

return iceTransport;
}

private getTransportByMLineIndex(index: number) {
const sdp = this.buildOfferSdp();
const media = sdp.media[index];
if (!media) {
return;
}
const transport = this.getTransportByMid(media.rtp.muxId!);

return transport;
}

async addIceCandidate(candidateMessage: RTCIceCandidate) {
const candidate = IceCandidate.fromJSON(candidateMessage);
let iceTransport: RTCIceTransport;
if (typeof candidateMessage.sdpMLineIndex === "number") {
iceTransport =
this.transceivers[candidateMessage.sdpMLineIndex]?.dtlsTransport
?.iceTransport;
} else {
if (!candidate) {
return;
}

let iceTransport: RTCIceTransport | undefined;

if (typeof candidate.sdpMid === "number") {
iceTransport = this.getTransportByMid(candidate.sdpMid);
}

if (!iceTransport && typeof candidate.sdpMLineIndex === "number") {
iceTransport = this.getTransportByMLineIndex(candidate.sdpMLineIndex);
}

if (!iceTransport) {
iceTransport = this.iceTransports[0];
}

if (iceTransport) {
await iceTransport.addRemoteCandidate(candidate);
} else {
log("iceTransport not found for sdpMLineIndex", candidate);
log("iceTransport not found", candidate);
}
}

Expand Down Expand Up @@ -714,6 +765,7 @@ export class RTCPeerConnection extends EventTarget {
transceiver = this.addTransceiver(remoteMedia.kind, {
direction: "recvonly",
});
transceiver.mid = remoteMedia.rtp.muxId;
this.onRemoteTransceiverAdded.execute(transceiver);
}

Expand All @@ -731,6 +783,7 @@ export class RTCPeerConnection extends EventTarget {
} else if (remoteMedia.kind === "application") {
if (!this.sctpTransport) {
this.sctpTransport = this.createSctpTransport();
this.sctpTransport.mid = remoteMedia.rtp.muxId;
}

if (bundle) {
Expand All @@ -743,7 +796,7 @@ export class RTCPeerConnection extends EventTarget {

dtlsTransport = this.sctpTransport.dtlsTransport;

this.setRemoteSCTP(remoteMedia, this.sctpTransport);
this.setRemoteSCTP(remoteMedia, this.sctpTransport, i);
} else {
throw new Error("invalid media kind");
}
Expand Down Expand Up @@ -888,7 +941,8 @@ export class RTCPeerConnection extends EventTarget {

private setRemoteSCTP(
remoteMedia: MediaDescription,
sctpTransport: RTCSctpTransport
sctpTransport: RTCSctpTransport,
mLineIndex: number
) {
// # configure sctp
this.sctpRemotePort = remoteMedia.sctpPort;
Expand All @@ -897,6 +951,7 @@ export class RTCPeerConnection extends EventTarget {
}

sctpTransport.setRemotePort(this.sctpRemotePort);
sctpTransport.mLineIndex = mLineIndex;
if (!sctpTransport.mid) {
sctpTransport.mid = remoteMedia.rtp.muxId;
}
Expand Down Expand Up @@ -980,13 +1035,10 @@ export class RTCPeerConnection extends EventTarget {

const direction = options.direction || "sendrecv";

const dtlsTransport = this.createTransport(
[
SRTP_PROFILE.SRTP_AEAD_AES_128_GCM, // prefer
SRTP_PROFILE.SRTP_AES128_CM_HMAC_SHA1_80,
],
this.transceivers.length
);
const dtlsTransport = this.createTransport([
SRTP_PROFILE.SRTP_AEAD_AES_128_GCM, // prefer
SRTP_PROFILE.SRTP_AES128_CM_HMAC_SHA1_80,
]);

const sender = new RTCRtpSender(trackOrKind);
const receiver = new RTCRtpReceiver(kind, sender.ssrc);
Expand Down Expand Up @@ -1114,18 +1166,14 @@ export class RTCPeerConnection extends EventTarget {
media = createMediaDescriptionForTransceiver(
transceiver,
this.cname,
andDirection(transceiver.direction, transceiver.offerDirection),
transceiver.mid!
andDirection(transceiver.direction, transceiver.offerDirection)
);
dtlsTransport = transceiver.dtlsTransport;
} else if (remoteMedia.kind === "application") {
if (!this.sctpTransport || !this.sctpTransport.mid) {
throw new Error("sctpTransport not found");
}
media = createMediaDescriptionForSctp(
this.sctpTransport,
this.sctpTransport.mid
);
media = createMediaDescriptionForSctp(this.sctpTransport);

dtlsTransport = this.sctpTransport.dtlsTransport;
} else {
Expand Down Expand Up @@ -1295,8 +1343,7 @@ export class RTCPeerConnection extends EventTarget {
export function createMediaDescriptionForTransceiver(
transceiver: RTCRtpTransceiver,
cname: string,
direction: Direction,
mid: string
direction: Direction
) {
const media = new MediaDescription(
transceiver.kind,
Expand All @@ -1309,7 +1356,7 @@ export function createMediaDescriptionForTransceiver(
media.rtp = {
codecs: transceiver.codecs,
headerExtensions: transceiver.headerExtensions,
muxId: mid,
muxId: transceiver.mid,
};
media.rtcpHost = "0.0.0.0";
media.rtcpPort = 9;
Expand Down Expand Up @@ -1338,18 +1385,15 @@ export function createMediaDescriptionForTransceiver(
return media;
}

export function createMediaDescriptionForSctp(
sctp: RTCSctpTransport,
mid: string
) {
export function createMediaDescriptionForSctp(sctp: RTCSctpTransport) {
const media = new MediaDescription(
"application",
DISCARD_PORT,
"UDP/DTLS/SCTP",
["webrtc-datachannel"]
);
media.sctpPort = sctp.port;
media.rtp.muxId = mid;
media.rtp.muxId = sctp.mid;
media.sctpCapabilities = RTCSctpTransport.getCapabilities();

addTransportDescription(media, sctp.dtlsTransport);
Expand Down
10 changes: 10 additions & 0 deletions packages/webrtc/src/transport/ice.ts
View file
@@ -1,9 +1,11 @@
import Event from "rx.mini";
import { v4 } from "uuid";

import { Candidate, Connection, IceOptions } from "../../../ice/src";
import { candidateFromSdp, candidateToSdp } from "../sdp";

export class RTCIceTransport {
readonly id = v4();
connection = this.gather.connection;
state: RTCIceConnectionState = "new";

Expand Down Expand Up @@ -182,6 +184,14 @@ export class RTCIceCandidate {
static isThis(o: any) {
if (typeof o?.candidate === "string") return true;
}

toJSON() {
return {
candidate: this.candidate,
sdpMid: this.sdpMid,
sdpMLineIndex: this.sdpMLineIndex,
};
}
}

export class IceCandidate {
Expand Down
3 changes: 2 additions & 1 deletion packages/webrtc/src/transport/sctp.ts
View file
Expand Up @@ -28,9 +28,10 @@ export class RTCSctpTransport {
sctp!: SCTP;

readonly onDataChannel = new Event<[RTCDataChannel]>();
readonly uuid = uuid.v4();
readonly id = uuid.v4();

mid?: string;
mLineIndex?: number;
bundled = false;
dataChannels: { [key: number]: RTCDataChannel } = {};

Expand Down
6 changes: 3 additions & 3 deletions packages/webrtc/tests/issue/142.test.ts
View file
Expand Up @@ -20,7 +20,7 @@ a=ice-ufrag:KFPTBCP4YKPZTY1Q
a=ice-pwd:VA0L9MQMWS8IYO9NJF+ITE++
a=fingerprint:sha-256 92:D6:06:D6:CB:64:B4:EF:47:76:00:F7:48:E0:ED:DD:9F:3E:DA:16:41:49:47:43:E0:77:DD:C6:D7:83:7E:19
a=setup:passive
a=mid:0
a=mid:0_srtp
a=sendrecv
a=msid:virtual-6666 virtual-6666
a=rtcp-mux
Expand All @@ -34,7 +34,7 @@ a=ice-ufrag:KFPTBCP4YKPZTY1Q
a=ice-pwd:VA0L9MQMWS8IYO9NJF+ITE++
a=fingerprint:sha-256 92:D6:06:D6:CB:64:B4:EF:47:76:00:F7:48:E0:ED:DD:9F:3E:DA:16:41:49:47:43:E0:77:DD:C6:D7:83:7E:19
a=setup:passive
a=mid:1
a=mid:1_srtp
a=sendrecv
a=msid:13945094204333313074/3212739396 13945094204333313074/3212739396
a=rtcp-mux
Expand All @@ -50,7 +50,7 @@ a=ice-ufrag:KFPTBCP4YKPZTY1Q
a=ice-pwd:VA0L9MQMWS8IYO9NJF+ITE++
a=fingerprint:sha-256 92:D6:06:D6:CB:64:B4:EF:47:76:00:F7:48:E0:ED:DD:9F:3E:DA:16:41:49:47:43:E0:77:DD:C6:D7:83:7E:19
a=setup:passive
a=mid:2
a=mid:2_sctp
a=sctpmap:5000 webrtc-datachannel 1024
`;

Expand Down

0 comments on commit 32a3610

Please sign in to comment.