From a4e853e1d47428a53dc16b30a598bf4680e74852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 6 May 2021 17:15:04 +0200 Subject: [PATCH 01/53] Add types for MSC3077 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callEventTypes.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index dce14648533..f5b81ab5d27 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -1,11 +1,22 @@ // allow camelcase as these are events type that go onto the wire /* eslint-disable camelcase */ +// TODO: Change to "sdp_stream_metadata" when MSC3077 is merged +export const SDPStreamMetadataKey = "org.matrix.msc3077.sdp_stream_metadata"; + export enum SDPStreamMetadataPurpose { Usermedia = "m.usermedia", Screenshare = "m.screenshare", } +export interface SDPStreamMetadataObject { + purpose: SDPStreamMetadataPurpose, +} + +export interface SDPStreamMetadata { + [key: string]: SDPStreamMetadataObject, +} + interface CallOfferAnswer { type: string; sdp: string; @@ -18,6 +29,7 @@ export interface CallCapabilities { export interface MCallAnswer { answer: CallOfferAnswer; capabilities: CallCapabilities; + [SDPStreamMetadataKey]: SDPStreamMetadata; } export interface MCallOfferNegotiate { @@ -25,6 +37,7 @@ export interface MCallOfferNegotiate { description: CallOfferAnswer; lifetime: number; capabilities: CallCapabilities; + [SDPStreamMetadataKey]: SDPStreamMetadata; } export interface MCallReplacesTarget { From 631faa2046704ee6c742e6db9876a47ecf39f74a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 6 May 2021 17:18:30 +0200 Subject: [PATCH 02/53] Send SDPStreamMetadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d84be56b784..1c7cb2bda52 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -34,6 +34,8 @@ import { MCallOfferNegotiate, CallCapabilities, SDPStreamMetadataPurpose, + SDPStreamMetadata, + SDPStreamMetadataKey, } from './callEventTypes'; import { CallFeed } from './callFeed'; @@ -448,6 +450,20 @@ export class MatrixCall extends EventEmitter { return this.feeds.filter((feed) => {return !feed.isLocal()}); } + /** + * Generates and returns localSDPStreamMetadata + * @returns {SDPStreamMetadata} localSDPStreamMetadata + */ + private getLocalSDPStreamMetadata(): SDPStreamMetadata { + const metadata = {}; + this.getLocalFeeds().map((localFeed) => { + metadata[localFeed.stream.id] = { + purpose: localFeed.purpose, + }; + }); + return metadata; + } + /** * Returns true if there are no incoming feeds, * otherwise returns false @@ -812,7 +828,8 @@ export class MatrixCall extends EventEmitter { logger.debug( "Setting screen sharing stream to the local video element", ); - this.pushNewFeed(this.screenSharingStream, this.client.getUserId(), SDPStreamMetadataPurpose.Screenshare); + // XXX: Because this is to support backwards compatibility we pretend like this stream is usermedia + this.pushNewFeed(this.screenSharingStream, this.client.getUserId(), SDPStreamMetadataPurpose.Usermedia); } else { this.pushNewFeed(stream, this.client.getUserId(), SDPStreamMetadataPurpose.Usermedia); } @@ -1196,6 +1213,8 @@ export class MatrixCall extends EventEmitter { } } + content[SDPStreamMetadataKey] = this.getLocalSDPStreamMetadata(); + // Get rid of any candidates waiting to be sent: they'll be included in the local // description we just got and will send in the offer. logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in offer`); From 6a920fe6232dc7566aa88c4d265989e367d7e172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 07:56:27 +0200 Subject: [PATCH 03/53] Get sdpStreamMetadata from invite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 1c7cb2bda52..d5e4891fa46 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -310,6 +310,8 @@ export class MatrixCall extends EventEmitter { private remoteAssertedIdentity: AssertedIdentity; + private remoteSDPStreamMetadata: SDPStreamMetadata; + constructor(opts: CallOpts) { super(); this.roomId = opts.roomId; @@ -528,6 +530,13 @@ export class MatrixCall extends EventEmitter { logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); } + const sdpStreamMetadata = invite[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.remoteSDPStreamMetadata = sdpStreamMetadata; + } else { + logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + } + this.peerConn = this.createPeerConnection(); // we must set the party ID before await-ing on anything: the call event // handler will start giving us more call events (eg. candidates) so if From 4d4a6ede21bd122b821f68ff27bc344161fa8a17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 10:41:59 +0200 Subject: [PATCH 04/53] Use somicolons instead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callEventTypes.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index f5b81ab5d27..5325678438b 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -10,11 +10,11 @@ export enum SDPStreamMetadataPurpose { } export interface SDPStreamMetadataObject { - purpose: SDPStreamMetadataPurpose, + purpose: SDPStreamMetadataPurpose; } export interface SDPStreamMetadata { - [key: string]: SDPStreamMetadataObject, + [key: string]: SDPStreamMetadataObject; } interface CallOfferAnswer { From 23f5c2e03f0bbbed35aa39d0b2e3071b2d65bde0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 10:43:36 +0200 Subject: [PATCH 05/53] Add a separate method to push local feed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d5e4891fa46..dfda7aadde4 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -485,6 +485,18 @@ export class MatrixCall extends EventEmitter { this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId)); this.emit(CallEvent.FeedsChanged, this.feeds); } + + private pushLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose) { + const userId = this.client.getUserId(); + + // We try to replace an existing feed if there already is one with the same purpose + const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose); + if (existingFeed) { + existingFeed.setNewStream(stream); + } else { + this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId)); + this.emit(CallEvent.FeedsChanged, this.feeds); + } } private deleteAllFeeds() { @@ -838,9 +850,9 @@ export class MatrixCall extends EventEmitter { "Setting screen sharing stream to the local video element", ); // XXX: Because this is to support backwards compatibility we pretend like this stream is usermedia - this.pushNewFeed(this.screenSharingStream, this.client.getUserId(), SDPStreamMetadataPurpose.Usermedia); + this.pushLocalFeed(this.screenSharingStream, SDPStreamMetadataPurpose.Usermedia); } else { - this.pushNewFeed(stream, this.client.getUserId(), SDPStreamMetadataPurpose.Usermedia); + this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); } // why do we enable audio (and only audio) tracks here? -- matthew @@ -910,7 +922,7 @@ export class MatrixCall extends EventEmitter { return; } - this.pushNewFeed(stream, this.client.getUserId(), SDPStreamMetadataPurpose.Usermedia); + this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); this.localAVStream = stream; logger.info("Got local AV stream with id " + this.localAVStream.id); From cebdc44689da34fd67dc9798013e98c2a0d39db5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 11:11:31 +0200 Subject: [PATCH 06/53] Set remoteSDPStreamMetadata from answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index dfda7aadde4..c5150a5d4ed 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1060,6 +1060,13 @@ export class MatrixCall extends EventEmitter { this.setState(CallState.Connecting); + const sdpStreamMetadata = event.getContent()[SDPStreamMetadataKey]; + if (sdpStreamMetadata) { + this.remoteSDPStreamMetadata = sdpStreamMetadata; + } else { + logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + } + try { await this.peerConn.setRemoteDescription(event.getContent().answer); } catch (e) { From 25eb6de22039942f9fae61f64f68752e22ec988a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 11:17:08 +0200 Subject: [PATCH 07/53] Send SDPStreamMetadata in answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index c5150a5d4ed..30f19d892fb 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -878,6 +878,7 @@ export class MatrixCall extends EventEmitter { // required to still be sent for backwards compat type: this.peerConn.localDescription.type, }, + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), } as MCallAnswer; if (this.client._supportsCallTransfer) { From 30f22634439a8f6cb3ec98932a75bc73b70331be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 11:18:25 +0200 Subject: [PATCH 08/53] Rework pushing of remote feeds for MSC3077 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 75 ++++++++++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 30f19d892fb..7c1961cb395 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -475,7 +475,52 @@ export class MatrixCall extends EventEmitter { return !this.feeds.some((feed) => !feed.isLocal()); } - private pushNewFeed(stream: MediaStream, userId: string, purpose: SDPStreamMetadataPurpose) { + private pushRemoteFeed(stream: MediaStream) { + // Fallback to old behavior if the other side doesn't support SDPStreamMetadata + if (!this.remoteSDPStreamMetadata) { + this.pushRemoteFeedWithoutMetadata(stream); + return; + } + + const userId = this.getOpponentMember().userId; + const streamMetadata = this.remoteSDPStreamMetadata[stream.id]; + + if (!streamMetadata) { + logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); + return; + } + + // Try to find a feed with the same purpose as the new stream, + // if we find it replace the old stream with the new one + const existingFeed = this.getRemoteFeeds().find((feed) => feed.purpose === streamMetadata.purpose); + if (existingFeed) { + existingFeed.setNewStream(stream); + } else { + this.feeds.push(new CallFeed(stream, userId, streamMetadata.purpose, this.client, this.roomId)); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + + logger.info(`Pushed remote stream with id ${stream.id}. Stream active? ${stream.active}`); + } + + /** + * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata + */ + private pushRemoteFeedWithoutMetadata(stream: MediaStream) { + const userId = this.getOpponentMember().userId; + // We can guess the purpose here since the other client can only send one stream + const purpose = SDPStreamMetadataPurpose.Usermedia; + const oldRemoteStream = this.feeds.find((feed) => {return !feed.isLocal()})?.stream; + + // Note that we check by ID and always set the remote stream: Chrome appears + // to make new stream objects when transceiver directionality is changed and the 'active' + // status of streams change - Dave + // If we already have a stream, check this stream has the same id + if (oldRemoteStream && stream.id !== oldRemoteStream.id) { + logger.warn(`Ignoring new stream ID ${stream.id}: we already have stream ID ${oldRemoteStream.id}`); + return; + } + // Try to find a feed with the same stream id as the new stream, // if we find it replace the old stream with the new one const feed = this.feeds.find((feed) => feed.stream.id === stream.id); @@ -486,6 +531,9 @@ export class MatrixCall extends EventEmitter { this.emit(CallEvent.FeedsChanged, this.feeds); } + logger.info(`Pushed remote stream with id ${stream.id}. Stream active? ${stream.active}`); + } + private pushLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose) { const userId = this.client.getUserId(); @@ -1348,30 +1396,7 @@ export class MatrixCall extends EventEmitter { return; } - const oldRemoteStream = this.feeds.find((feed) => {return !feed.isLocal()})?.stream; - - // If we already have a stream, check this track is from the same one - // Note that we check by ID and always set the remote stream: Chrome appears - // to make new stream objects when tranciever directionality is changed and the 'active' - // status of streams change - Dave - if (oldRemoteStream && ev.streams[0].id !== oldRemoteStream.id) { - logger.warn( - `Ignoring new stream ID ${ev.streams[0].id}: we already have stream ID ${oldRemoteStream.id}`, - ); - return; - } - - if (!oldRemoteStream) { - logger.info("Got remote stream with id " + ev.streams[0].id); - } - - const newRemoteStream = ev.streams[0]; - - logger.debug(`Track id ${ev.track.id} of kind ${ev.track.kind} added`); - - this.pushNewFeed(newRemoteStream, this.getOpponentMember().userId, SDPStreamMetadataPurpose.Usermedia) - - logger.info("playing remote. stream active? " + newRemoteStream.active); + this.pushRemoteFeed(ev.streams[0]); }; onNegotiationNeeded = async () => { From 50e0f6353a3329060edf79d3e1eec881916e8374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 13:15:09 +0200 Subject: [PATCH 09/53] Move adding tracks into pushLocalFeed() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 7c1961cb395..5e933581614 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -545,6 +545,14 @@ export class MatrixCall extends EventEmitter { this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId)); this.emit(CallEvent.FeedsChanged, this.feeds); } + + // why do we enable audio (and only audio) tracks here? -- matthew + setTracksEnabled(stream.getAudioTracks(), true); + + for (const track of stream.getTracks()) { + logger.info(`Adding track with id ${track.id} and with kind ${track.kind} to peer connection`) + this.peerConn.addTrack(track, stream); + } } private deleteAllFeeds() { @@ -903,18 +911,6 @@ export class MatrixCall extends EventEmitter { this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); } - // why do we enable audio (and only audio) tracks here? -- matthew - setTracksEnabled(stream.getAudioTracks(), true); - - for (const audioTrack of stream.getAudioTracks()) { - logger.info("Adding audio track with id " + audioTrack.id); - this.peerConn.addTrack(audioTrack, stream); - } - for (const videoTrack of (this.screenSharingStream || stream).getVideoTracks()) { - logger.info("Adding video track with id " + videoTrack.id); - this.peerConn.addTrack(videoTrack, stream); - } - // Now we wait for the negotiationneeded event }; @@ -975,10 +971,6 @@ export class MatrixCall extends EventEmitter { this.localAVStream = stream; logger.info("Got local AV stream with id " + this.localAVStream.id); - setTracksEnabled(stream.getAudioTracks(), true); - for (const track of stream.getTracks()) { - this.peerConn.addTrack(track, stream); - } this.setState(CallState.CreateAnswer); From 640d13af999190b736f6019779e0c4fad4c18c6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 13:20:36 +0200 Subject: [PATCH 10/53] Set remoteSDPStreamMetadata in onNegotiateReceived() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 5e933581614..90901b48a3b 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1184,6 +1184,13 @@ export class MatrixCall extends EventEmitter { this.unholdingRemote = false; } + const metadata = event.getContent()[SDPStreamMetadataKey]; + if (metadata) { + this.remoteSDPStreamMetadata = metadata; + } else { + logger.warn("Received negotiation event without SDPStreamMetadata!") + } + try { await this.peerConn.setRemoteDescription(description); From b1ace49f9a2a855da3426ddb77e5133b8ce9320e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 20:53:19 +0200 Subject: [PATCH 11/53] Add methods useful for screensharing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 90901b48a3b..06db92be497 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -762,6 +762,23 @@ export class MatrixCall extends EventEmitter { this.sendVoipEvent(EventType.CallReject, {}); } + /** + * Returns true if this.remoteSDPStreamMetadata is defined, otherwise returns false + * @returns {boolean} can screenshare + */ + public opponentSupportsSDPStreamMetadata(): boolean { + return Boolean(this.remoteSDPStreamMetadata); + } + + /** + * If there is a screensharing stream returns true, otherwise returns false + * @returns {boolean} is screensharing + */ + public isScreensharing(): boolean { + console.log("LOG stream", this.screenSharingStream); + return Boolean(this.screenSharingStream); + } + /** * Set whether our outbound video should be muted or not. * @param {boolean} muted True to mute the outbound video. From 4d74b5cdad228c40e0d5108a4445a8c52a228be1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 20:57:45 +0200 Subject: [PATCH 12/53] Send SDPStreamMetadata in negotiation response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 06db92be497..836ff29480d 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1230,6 +1230,7 @@ export class MatrixCall extends EventEmitter { this.sendVoipEvent(EventType.CallNegotiate, { description: this.peerConn.localDescription, + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), }); } } catch (err) { From 8c8a68d3ae7ba5e1e59f9cb9ef0336604ddfa7a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 21:01:56 +0200 Subject: [PATCH 13/53] Add methods to delete feeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 836ff29480d..1daab7f76fe 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -560,6 +560,42 @@ export class MatrixCall extends EventEmitter { this.emit(CallEvent.FeedsChanged, this.feeds); } + private deleteLocalFeedByStream(stream: MediaStream) { + logger.debug(`Removing feed with stream id ${stream.id}`); + + const feed = this.getLocalFeeds().find((feed) => feed.stream.id === stream.id); + if (!feed) { + logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); + return; + } + + this.feeds.splice(this.feeds.indexOf(feed), 1); + this.emit(CallEvent.FeedsChanged, this.feeds); + + for (const track of stream.getTracks()) { + // XXX: This is ugly and there has to be a way to do this more nicely + for (const sender of this.peerConn.getSenders()) { + if (sender.track?.id === track.id) { + this.peerConn.removeTrack(sender); + } + } + track.stop(); + } + } + + private deleteRemoteFeedByStream(stream: MediaStream) { + logger.debug(`Removing feed with stream id ${stream.id}`); + + const feed = this.getRemoteFeeds().find((feed) => feed.stream.id === stream.id); + if (!feed) { + logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); + return; + } + + this.feeds.splice(this.feeds.indexOf(feed), 1); + this.emit(CallEvent.FeedsChanged, this.feeds); + } + // The typescript definitions have this type as 'any' :( public async getCurrentCallStats(): Promise { if (this.callHasEnded()) { From 0e6b43a7692dc8dc8de75510bef12765ae9e23af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 21:02:56 +0200 Subject: [PATCH 14/53] Hook up methods to delete feeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 1daab7f76fe..a1dec5762ee 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1449,7 +1449,9 @@ export class MatrixCall extends EventEmitter { return; } - this.pushRemoteFeed(ev.streams[0]); + const stream = ev.streams[0]; + this.pushRemoteFeed(stream); + stream.addEventListener("removetrack", () => this.deleteRemoteFeedByStream(stream)); }; onNegotiationNeeded = async () => { From a35559be65c0852225bab1c18d331dd4d1edd5da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 7 May 2021 21:33:05 +0200 Subject: [PATCH 15/53] Add a method to start screensharing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 59 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index a1dec5762ee..d8dd8aabf21 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -815,6 +815,65 @@ export class MatrixCall extends EventEmitter { return Boolean(this.screenSharingStream); } + /** + * Starts/stops screensharing + * @param enabled the desired screensharing state + * @param selectDesktopCapturerSource callBack to select a screensharing stream on desktop + * @returns {boolean} new screensharing state + */ + public async setScreensharingEnabled( + enabled: boolean, + selectDesktopCapturerSource?: () => Promise, + ) { + if (!this.opponentSupportsSDPStreamMetadata) { + logger.warn("The other side doesn't support multiple stream, therefore we can't screenshare"); + // TODO: Use replaceTrack() + return false; + } + + logger.debug(`Set screensharing enabled? ${enabled}`); + if (enabled) { + if (this.isScreensharing()) { + logger.warn(`There is already a screensharing stream - there is nothing to do!`); + return false; + } + + try { + const screenshareConstraints = await getScreenshareContraints(selectDesktopCapturerSource); + if (!screenshareConstraints) { + logger.error("Failed to get screensharing constraints!"); + return; + } + + if (window.electron?.getDesktopCapturerSources) { + // We are using Electron + logger.debug("Getting screen stream using getUserMedia()..."); + this.screenSharingStream = await navigator.mediaDevices.getUserMedia(screenshareConstraints); + } else { + // We are not using Electron + logger.debug("Getting screen stream using getDisplayMedia()..."); + this.screenSharingStream = await navigator.mediaDevices.getDisplayMedia(screenshareConstraints); + } + + this.pushLocalFeed(this.screenSharingStream, SDPStreamMetadataPurpose.Screenshare); + return true; + } catch (err) { + this.emit(CallEvent.Error, + new CallError(CallErrorCode.NoUserMedia, "Failed to get screen-sharing stream: ", err), + ); + return false; + } + } else { + if (!this.isScreensharing()) { + logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); + return false; + } + this.deleteLocalFeedByStream(this.screenSharingStream); + this.screenSharingStream = null; + return false; + } + } + /** * Set whether our outbound video should be muted or not. * @param {boolean} muted True to mute the outbound video. From 972aef7a9df60c9bb3cd0ffa2e125ea414b8badf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 07:40:46 +0200 Subject: [PATCH 16/53] Merge feed delete methods and add sender arrays MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 52 +++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d8dd8aabf21..59670933c7c 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -269,10 +269,14 @@ export class MatrixCall extends EventEmitter { private peerConn: RTCPeerConnection; private feeds: Array; private screenSharingStream: MediaStream; + // TODO: Rename to usermedia rather than AV for consistency private localAVStream: MediaStream; + private usermediaSenders: Array; + private screensharingSenders: Array; private inviteOrAnswerSent: boolean; private waitForLocalAVStream: boolean; // XXX: I don't know why this is called 'config'. + // XXX: Do we even needs this? Seems to be unused private config: MediaStreamConstraints; private successor: MatrixCall; private opponentMember: RoomMember; @@ -349,6 +353,9 @@ export class MatrixCall extends EventEmitter { this.vidMuted = false; this.feeds = []; + + this.usermediaSenders = []; + this.screensharingSenders = []; } /** @@ -549,9 +556,14 @@ export class MatrixCall extends EventEmitter { // why do we enable audio (and only audio) tracks here? -- matthew setTracksEnabled(stream.getAudioTracks(), true); + const senderArray = purpose === SDPStreamMetadataPurpose.Usermedia ? + this.usermediaSenders : this.screensharingSenders; + // Empty the array + senderArray.splice(0, senderArray.length); + for (const track of stream.getTracks()) { logger.info(`Adding track with id ${track.id} and with kind ${track.kind} to peer connection`) - this.peerConn.addTrack(track, stream); + senderArray.push(this.peerConn.addTrack(track, stream)); } } @@ -560,33 +572,10 @@ export class MatrixCall extends EventEmitter { this.emit(CallEvent.FeedsChanged, this.feeds); } - private deleteLocalFeedByStream(stream: MediaStream) { + private deleteFeedByStream(stream: MediaStream) { logger.debug(`Removing feed with stream id ${stream.id}`); - const feed = this.getLocalFeeds().find((feed) => feed.stream.id === stream.id); - if (!feed) { - logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); - return; - } - - this.feeds.splice(this.feeds.indexOf(feed), 1); - this.emit(CallEvent.FeedsChanged, this.feeds); - - for (const track of stream.getTracks()) { - // XXX: This is ugly and there has to be a way to do this more nicely - for (const sender of this.peerConn.getSenders()) { - if (sender.track?.id === track.id) { - this.peerConn.removeTrack(sender); - } - } - track.stop(); - } - } - - private deleteRemoteFeedByStream(stream: MediaStream) { - logger.debug(`Removing feed with stream id ${stream.id}`); - - const feed = this.getRemoteFeeds().find((feed) => feed.stream.id === stream.id); + const feed = this.feeds.find((feed) => feed.stream.id === stream.id); if (!feed) { logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); return; @@ -868,7 +857,14 @@ export class MatrixCall extends EventEmitter { logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); return false; } - this.deleteLocalFeedByStream(this.screenSharingStream); + + for (const sender of this.screensharingSenders) { + this.peerConn.removeTrack(sender); + } + for (const track of this.screenSharingStream.getTracks()) { + track.stop(); + } + this.deleteFeedByStream(this.screenSharingStream); this.screenSharingStream = null; return false; } @@ -1510,7 +1506,7 @@ export class MatrixCall extends EventEmitter { const stream = ev.streams[0]; this.pushRemoteFeed(stream); - stream.addEventListener("removetrack", () => this.deleteRemoteFeedByStream(stream)); + stream.addEventListener("removetrack", () => this.deleteFeedByStream(stream)); }; onNegotiationNeeded = async () => { From 377ca0c6786ebffeb7b541d3605801b9c6f2d5c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 08:01:56 +0200 Subject: [PATCH 17/53] Add getScreensharingStream() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 59670933c7c..42fe3429729 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1875,6 +1875,26 @@ export class MatrixCall extends EventEmitter { } } +async function getScreensharingStream( + selectDesktopCapturerSource?: () => Promise, +): Promise { + const screenshareConstraints = await getScreenshareContraints(selectDesktopCapturerSource); + if (!screenshareConstraints) { + logger.error("Failed to get screensharing constraints!"); + return; + } + + if (window.electron?.getDesktopCapturerSources) { + // We are using Electron + logger.debug("Getting screen stream using getUserMedia()..."); + return await navigator.mediaDevices.getUserMedia(screenshareConstraints); + } else { + // We are not using Electron + logger.debug("Getting screen stream using getDisplayMedia()..."); + return await navigator.mediaDevices.getDisplayMedia(screenshareConstraints); + } +} + function setTracksEnabled(tracks: Array, enabled: boolean) { for (let i = 0; i < tracks.length; i++) { tracks[i].enabled = enabled; From e9b802deb3e5c9d9dcdba0c1e82b8a8268f782d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 08:03:45 +0200 Subject: [PATCH 18/53] Use getScreensharingStream() in setScreensharingEnabled() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 42fe3429729..b09ed8aae4b 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -824,15 +824,37 @@ export class MatrixCall extends EventEmitter { if (enabled) { if (this.isScreensharing()) { logger.warn(`There is already a screensharing stream - there is nothing to do!`); - return false; + return true; } try { - const screenshareConstraints = await getScreenshareContraints(selectDesktopCapturerSource); - if (!screenshareConstraints) { - logger.error("Failed to get screensharing constraints!"); - return; - } + this.screenSharingStream = await getScreensharingStream(selectDesktopCapturerSource); + if (!this.screenSharingStream) return false; + this.pushLocalFeed(this.screenSharingStream, SDPStreamMetadataPurpose.Screenshare); + return true; + } catch (err) { + this.emit(CallEvent.Error, + new CallError(CallErrorCode.NoUserMedia, "Failed to get screen-sharing stream: ", err), + ); + return false; + } + } else { + if (!this.isScreensharing()) { + logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); + return false; + } + + for (const sender of this.screensharingSenders) { + this.peerConn.removeTrack(sender); + } + for (const track of this.screenSharingStream.getTracks()) { + track.stop(); + } + this.deleteFeedByStream(this.screenSharingStream); + this.screenSharingStream = null; + return false; + } + } if (window.electron?.getDesktopCapturerSources) { // We are using Electron From cbc74815d89c610d88e9215131702299a6d35aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 08:46:29 +0200 Subject: [PATCH 19/53] Use getScreensharingStream() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index b09ed8aae4b..72908989684 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -392,22 +392,12 @@ export class MatrixCall extends EventEmitter { logger.debug("placeScreenSharingCall"); this.checkForErrorListener(); try { - const screenshareConstraints = await getScreenshareContraints(selectDesktopCapturerSource); - if (!screenshareConstraints) { + this.screenSharingStream = await getScreensharingStream(selectDesktopCapturerSource); + if (!this.screenSharingStream) { this.terminate(CallParty.Local, CallErrorCode.NoUserMedia, false); return; } - if (window.electron?.getDesktopCapturerSources) { - // We are using Electron - logger.debug("Getting screen stream using getUserMedia()..."); - this.screenSharingStream = await navigator.mediaDevices.getUserMedia(screenshareConstraints); - } else { - // We are not using Electron - logger.debug("Getting screen stream using getDisplayMedia()..."); - this.screenSharingStream = await navigator.mediaDevices.getDisplayMedia(screenshareConstraints); - } - logger.debug("Got screen stream, requesting audio stream..."); const audioConstraints = getUserMediaContraints(ConstraintsType.Audio); this.placeCallWithConstraints(audioConstraints); From d250e7387cdf1559749a61f7aca2a06f71b38235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 09:02:01 +0200 Subject: [PATCH 20/53] Merge screenshare track into usermedia stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 72908989684..9677f6e619f 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1025,8 +1025,11 @@ export class MatrixCall extends EventEmitter { logger.debug( "Setting screen sharing stream to the local video element", ); + const videoTrack = this.screenSharingStream.getTracks().find((track) => track.kind === "video"); + this.localAVStream.addTrack(videoTrack); + // XXX: Because this is to support backwards compatibility we pretend like this stream is usermedia - this.pushLocalFeed(this.screenSharingStream, SDPStreamMetadataPurpose.Usermedia); + this.pushLocalFeed(this.localAVStream, SDPStreamMetadataPurpose.Usermedia); } else { this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); } From 82c530da95d7ecf28f6e9015425905dec0db6e67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 09:08:06 +0200 Subject: [PATCH 21/53] Add setScreensharingEnabledWithoutMetadataSupport as a fallback() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 56 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 9677f6e619f..cc4d7a95eda 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -805,9 +805,7 @@ export class MatrixCall extends EventEmitter { selectDesktopCapturerSource?: () => Promise, ) { if (!this.opponentSupportsSDPStreamMetadata) { - logger.warn("The other side doesn't support multiple stream, therefore we can't screenshare"); - // TODO: Use replaceTrack() - return false; + return await this.setScreensharingEnabledWithoutMetadataSupport(enabled, selectDesktopCapturerSource); } logger.debug(`Set screensharing enabled? ${enabled}`); @@ -846,17 +844,36 @@ export class MatrixCall extends EventEmitter { } } - if (window.electron?.getDesktopCapturerSources) { - // We are using Electron - logger.debug("Getting screen stream using getUserMedia()..."); - this.screenSharingStream = await navigator.mediaDevices.getUserMedia(screenshareConstraints); - } else { - // We are not using Electron - logger.debug("Getting screen stream using getDisplayMedia()..."); - this.screenSharingStream = await navigator.mediaDevices.getDisplayMedia(screenshareConstraints); - } + /** + * Starts/stops screensharing + * Should be used ONLY if the opponent doesn't support SDPStreamMetadata + * @param enabled the desired screensharing state + * @param selectDesktopCapturerSource callBack to select a screensharing stream on desktop + * @returns {boolean} new screensharing state + */ + private async setScreensharingEnabledWithoutMetadataSupport( + enabled: boolean, + selectDesktopCapturerSource?: () => Promise, + ) { + logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`); + if (enabled) { + if (this.isScreensharing()) { + logger.warn(`There is already a screensharing stream - there is nothing to do!`); + return true; + } + + try { + this.screenSharingStream = await getScreensharingStream(selectDesktopCapturerSource); + if (!this.screenSharingStream) return false; + + const videoScreensharingTrack = this.screenSharingStream.getTracks().find((track) => { + return track.kind === "video"; + }); + const userMediaVideoSender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === "video"; + }); + userMediaVideoSender.replaceTrack(videoScreensharingTrack); - this.pushLocalFeed(this.screenSharingStream, SDPStreamMetadataPurpose.Screenshare); return true; } catch (err) { this.emit(CallEvent.Error, @@ -870,14 +887,19 @@ export class MatrixCall extends EventEmitter { return false; } - for (const sender of this.screensharingSenders) { - this.peerConn.removeTrack(sender); - } + const videoScreensharingTrack = this.localAVStream.getTracks().find((track) => { + return track.kind === "video"; + }); + const userMediaVideoSender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === "video"; + }); + userMediaVideoSender.replaceTrack(videoScreensharingTrack); + for (const track of this.screenSharingStream.getTracks()) { track.stop(); } - this.deleteFeedByStream(this.screenSharingStream); this.screenSharingStream = null; + return false; } } From fc68bb3ae0256a5f37d606ed200dc21f58b5bde8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 09:29:31 +0200 Subject: [PATCH 22/53] Add ()!!! MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index cc4d7a95eda..bb302b0eea3 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -804,7 +804,7 @@ export class MatrixCall extends EventEmitter { enabled: boolean, selectDesktopCapturerSource?: () => Promise, ) { - if (!this.opponentSupportsSDPStreamMetadata) { + if (!this.opponentSupportsSDPStreamMetadata()) { return await this.setScreensharingEnabledWithoutMetadataSupport(enabled, selectDesktopCapturerSource); } From df28d87d253c11664c7e0acc3072240d703b4a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 09:29:41 +0200 Subject: [PATCH 23/53] Remove log line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index bb302b0eea3..f8b5796c3e6 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -790,7 +790,6 @@ export class MatrixCall extends EventEmitter { * @returns {boolean} is screensharing */ public isScreensharing(): boolean { - console.log("LOG stream", this.screenSharingStream); return Boolean(this.screenSharingStream); } @@ -893,6 +892,9 @@ export class MatrixCall extends EventEmitter { const userMediaVideoSender = this.usermediaSenders.find((sender) => { return sender.track?.kind === "video"; }); + if (!userMediaVideoSender) { + this.peerConn.removeTrack(userMediaVideoSender); + } userMediaVideoSender.replaceTrack(videoScreensharingTrack); for (const track of this.screenSharingStream.getTracks()) { From fa3b246de53f1e505b82bbf7723775590e9f2f5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 10:00:59 +0200 Subject: [PATCH 24/53] Add addToPeerConnection param MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index f8b5796c3e6..d7b5adab142 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -531,7 +531,7 @@ export class MatrixCall extends EventEmitter { logger.info(`Pushed remote stream with id ${stream.id}. Stream active? ${stream.active}`); } - private pushLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose) { + private pushLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true) { const userId = this.client.getUserId(); // We try to replace an existing feed if there already is one with the same purpose @@ -546,14 +546,16 @@ export class MatrixCall extends EventEmitter { // why do we enable audio (and only audio) tracks here? -- matthew setTracksEnabled(stream.getAudioTracks(), true); - const senderArray = purpose === SDPStreamMetadataPurpose.Usermedia ? - this.usermediaSenders : this.screensharingSenders; - // Empty the array - senderArray.splice(0, senderArray.length); + if (addToPeerConnection) { + const senderArray = purpose === SDPStreamMetadataPurpose.Usermedia ? + this.usermediaSenders : this.screensharingSenders; + // Empty the array + senderArray.splice(0, senderArray.length); - for (const track of stream.getTracks()) { - logger.info(`Adding track with id ${track.id} and with kind ${track.kind} to peer connection`) - senderArray.push(this.peerConn.addTrack(track, stream)); + for (const track of stream.getTracks()) { + logger.info(`Adding track with id ${track.id} and with kind ${track.kind} to peer connection`) + senderArray.push(this.peerConn.addTrack(track, stream)); + } } } From d0707e183d1dfa61449e4696b41c0cb69b3a8ba9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 10:02:27 +0200 Subject: [PATCH 25/53] Make shift-click work again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is VERY ugly but it works Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d7b5adab142..7950e14df86 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -399,7 +399,7 @@ export class MatrixCall extends EventEmitter { } logger.debug("Got screen stream, requesting audio stream..."); - const audioConstraints = getUserMediaContraints(ConstraintsType.Audio); + const audioConstraints = getUserMediaContraints(ConstraintsType.Video); this.placeCallWithConstraints(audioConstraints); } catch (err) { this.emit(CallEvent.Error, @@ -875,6 +875,8 @@ export class MatrixCall extends EventEmitter { }); userMediaVideoSender.replaceTrack(videoScreensharingTrack); + this.pushLocalFeed(this.screenSharingStream, SDPStreamMetadataPurpose.Screenshare, false); + return true; } catch (err) { this.emit(CallEvent.Error, @@ -888,20 +890,18 @@ export class MatrixCall extends EventEmitter { return false; } - const videoScreensharingTrack = this.localAVStream.getTracks().find((track) => { + const track = this.localAVStream.getTracks().find((track) => { return track.kind === "video"; }); - const userMediaVideoSender = this.usermediaSenders.find((sender) => { + const sender = this.usermediaSenders.find((sender) => { return sender.track?.kind === "video"; }); - if (!userMediaVideoSender) { - this.peerConn.removeTrack(userMediaVideoSender); - } - userMediaVideoSender.replaceTrack(videoScreensharingTrack); + sender.replaceTrack(track); for (const track of this.screenSharingStream.getTracks()) { track.stop(); } + this.deleteFeedByStream(this.screenSharingStream); this.screenSharingStream = null; return false; @@ -1051,11 +1051,17 @@ export class MatrixCall extends EventEmitter { logger.debug( "Setting screen sharing stream to the local video element", ); - const videoTrack = this.screenSharingStream.getTracks().find((track) => track.kind === "video"); - this.localAVStream.addTrack(videoTrack); - // XXX: Because this is to support backwards compatibility we pretend like this stream is usermedia + this.pushLocalFeed(this.screenSharingStream, SDPStreamMetadataPurpose.Screenshare, false); this.pushLocalFeed(this.localAVStream, SDPStreamMetadataPurpose.Usermedia); + + const videoScreensharingTrack = this.screenSharingStream.getTracks().find((track) => { + return track.kind === "video"; + }); + const userMediaVideoSender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === "video"; + }); + userMediaVideoSender.replaceTrack(videoScreensharingTrack); } else { this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); } From 7b333a34b56e99b124427d13226c990a3162eeee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 14:16:30 +0200 Subject: [PATCH 26/53] Warn level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 7950e14df86..8022df6e6c1 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1260,7 +1260,7 @@ export class MatrixCall extends EventEmitter { if (sdpStreamMetadata) { this.remoteSDPStreamMetadata = sdpStreamMetadata; } else { - logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); } try { From 18580624e6b9c9711d964e770adb0a66b3878200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 14:16:43 +0200 Subject: [PATCH 27/53] Use opponentSupportsSDPStreamMetadata() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 8022df6e6c1..b3210d87ad3 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -474,7 +474,7 @@ export class MatrixCall extends EventEmitter { private pushRemoteFeed(stream: MediaStream) { // Fallback to old behavior if the other side doesn't support SDPStreamMetadata - if (!this.remoteSDPStreamMetadata) { + if (!this.opponentSupportsSDPStreamMetadata()) { this.pushRemoteFeedWithoutMetadata(stream); return; } From c6764490c618e9a0916ed497e8b66e48cd1c2188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 14:17:10 +0200 Subject: [PATCH 28/53] Use for loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index b3210d87ad3..d2b33eb5388 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -455,11 +455,11 @@ export class MatrixCall extends EventEmitter { */ private getLocalSDPStreamMetadata(): SDPStreamMetadata { const metadata = {}; - this.getLocalFeeds().map((localFeed) => { + for (const localFeed of this.getLocalFeeds()) { metadata[localFeed.stream.id] = { purpose: localFeed.purpose, }; - }); + } return metadata; } From b67cd94ee2f4ac495885adc5c5ccffe339904bcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 15:39:11 +0200 Subject: [PATCH 29/53] Jest: should map SDPStreamMetadata to feeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/call.spec.ts | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index e5ef6de87f3..65f4e7a027b 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -16,6 +16,7 @@ limitations under the License. import {TestClient} from '../../TestClient'; import {MatrixCall, CallErrorCode, CallEvent} from '../../../src/webrtc/call'; +import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes'; const DUMMY_SDP = ( "v=0\r\n" + @@ -298,4 +299,33 @@ describe('Call', function() { // Hangup to stop timers call.hangup(CallErrorCode.UserHangup, true); }); + + it("should map SDPStreamMetadata to feeds", async () => { + const callPromise = call.placeVoiceCall(); + await client.httpBackend.flush(); + await callPromise; + + call.getOpponentMember = () => {return {userId: "@bob:bar.uk"}}; + + await call.onAnswerReceived({ + getContent: () => { + return { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, + }, + [SDPStreamMetadataKey]: { + "stream_id": { + purpose: SDPStreamMetadataPurpose.Usermedia, + } + } + }; + }, + }); + + call.pushRemoteFeed({id: "stream_id"}); + expect(call.getFeeds().find((feed) => feed.stream.id === "stream_id")?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia); + }); }); From 1e0d6b9d4aef1327b62d5761cf0088609cab7761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 15:50:20 +0200 Subject: [PATCH 30/53] Jest: should fallback to replaceTrack() if the other side doesn't support SPDStreamMetadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/call.spec.ts | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 65f4e7a027b..6a5955f1a35 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -17,6 +17,7 @@ limitations under the License. import {TestClient} from '../../TestClient'; import {MatrixCall, CallErrorCode, CallEvent} from '../../../src/webrtc/call'; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes'; +import { isMainThread } from 'worker_threads'; const DUMMY_SDP = ( "v=0\r\n" + @@ -328,4 +329,30 @@ describe('Call', function() { call.pushRemoteFeed({id: "stream_id"}); expect(call.getFeeds().find((feed) => feed.stream.id === "stream_id")?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia); }); + + it("should fallback to replaceTrack() if the other side doesn't support SPDStreamMetadata", async () => { + const callPromise = call.placeVoiceCall(); + await client.httpBackend.flush(); + await callPromise; + + call.getOpponentMember = () => {return {userId: "@bob:bar.uk"}}; + + await call.onAnswerReceived({ + getContent: () => { + return { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, + }, + }; + }, + }); + + call.setScreensharingEnabledWithoutMetadataSupport = jest.fn(); + + call.setScreensharingEnabled(true); + expect(call.setScreensharingEnabledWithoutMetadataSupport).toHaveBeenCalled(); + }); }); From 27c172361f059d90620d1d54215b7b19f855c0a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 8 May 2021 20:03:47 +0200 Subject: [PATCH 31/53] Add a log line to pushLocalFeed() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d2b33eb5388..0b583caffa7 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -557,6 +557,8 @@ export class MatrixCall extends EventEmitter { senderArray.push(this.peerConn.addTrack(track, stream)); } } + + logger.info(`Pushed local stream with id ${stream.id}. Stream active? ${stream.active}`); } private deleteAllFeeds() { From 16f569136b87d50389243dd1d978402c5b6ac146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 9 May 2021 18:48:08 +0200 Subject: [PATCH 32/53] Simplifie and avoid repetation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 0b583caffa7..4ed6058abc9 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -807,17 +807,22 @@ export class MatrixCall extends EventEmitter { enabled: boolean, selectDesktopCapturerSource?: () => Promise, ) { + // Skip if there is nothing to do + if (enabled && this.isScreensharing()) { + logger.warn(`There is already a screensharing stream - there is nothing to do!`); + return true; + } else if (!enabled && !this.isScreensharing()) { + logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); + return false; + } + + // Fallback to replaceTrack() if (!this.opponentSupportsSDPStreamMetadata()) { return await this.setScreensharingEnabledWithoutMetadataSupport(enabled, selectDesktopCapturerSource); } logger.debug(`Set screensharing enabled? ${enabled}`); if (enabled) { - if (this.isScreensharing()) { - logger.warn(`There is already a screensharing stream - there is nothing to do!`); - return true; - } - try { this.screenSharingStream = await getScreensharingStream(selectDesktopCapturerSource); if (!this.screenSharingStream) return false; @@ -830,11 +835,6 @@ export class MatrixCall extends EventEmitter { return false; } } else { - if (!this.isScreensharing()) { - logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); - return false; - } - for (const sender of this.screensharingSenders) { this.peerConn.removeTrack(sender); } @@ -860,11 +860,6 @@ export class MatrixCall extends EventEmitter { ) { logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`); if (enabled) { - if (this.isScreensharing()) { - logger.warn(`There is already a screensharing stream - there is nothing to do!`); - return true; - } - try { this.screenSharingStream = await getScreensharingStream(selectDesktopCapturerSource); if (!this.screenSharingStream) return false; @@ -887,11 +882,6 @@ export class MatrixCall extends EventEmitter { return false; } } else { - if (!this.isScreensharing()) { - logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); - return false; - } - const track = this.localAVStream.getTracks().find((track) => { return track.kind === "video"; }); From ff60bbac9d54883598302bc584dce270f5285537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 10 May 2021 07:35:25 +0200 Subject: [PATCH 33/53] Remove import that was a mistake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/call.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 6a5955f1a35..d1af1a77044 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -17,7 +17,6 @@ limitations under the License. import {TestClient} from '../../TestClient'; import {MatrixCall, CallErrorCode, CallEvent} from '../../../src/webrtc/call'; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes'; -import { isMainThread } from 'worker_threads'; const DUMMY_SDP = ( "v=0\r\n" + From a13cf0e1e05db9793f64a6cff75d846e3fd61969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 10 May 2021 12:27:21 +0200 Subject: [PATCH 34/53] Remove placeScreenSharingCall() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This method is quite problematic and doesn't have any benefits Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 4ed6058abc9..b22fbdff269 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -382,37 +382,6 @@ export class MatrixCall extends EventEmitter { await this.placeCallWithConstraints(constraints); } - /** - * Place a screen-sharing call to this room. This includes audio. - * This method is EXPERIMENTAL and subject to change without warning. It - * only works in Google Chrome and Firefox >= 44. - * @throws If you have not specified a listener for 'error' events. - */ - async placeScreenSharingCall(selectDesktopCapturerSource?: () => Promise) { - logger.debug("placeScreenSharingCall"); - this.checkForErrorListener(); - try { - this.screenSharingStream = await getScreensharingStream(selectDesktopCapturerSource); - if (!this.screenSharingStream) { - this.terminate(CallParty.Local, CallErrorCode.NoUserMedia, false); - return; - } - - logger.debug("Got screen stream, requesting audio stream..."); - const audioConstraints = getUserMediaContraints(ConstraintsType.Video); - this.placeCallWithConstraints(audioConstraints); - } catch (err) { - this.emit(CallEvent.Error, - new CallError( - CallErrorCode.NoUserMedia, - "Failed to get screen-sharing stream: ", err, - ), - ); - this.terminate(CallParty.Local, CallErrorCode.NoUserMedia, false); - } - this.type = CallType.Video; - } - public getOpponentMember() { return this.opponentMember; } From f2c215311f130163d3ce2091aa161b4f8e6a6bb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 10 May 2021 13:01:06 +0200 Subject: [PATCH 35/53] If we can't get constraints don't error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We do this because it could mean the user just hasn't selected a window/screen Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index b22fbdff269..07cf80ff5de 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1887,10 +1887,7 @@ async function getScreensharingStream( selectDesktopCapturerSource?: () => Promise, ): Promise { const screenshareConstraints = await getScreenshareContraints(selectDesktopCapturerSource); - if (!screenshareConstraints) { - logger.error("Failed to get screensharing constraints!"); - return; - } + if (!screenshareConstraints) return null; if (window.electron?.getDesktopCapturerSources) { // We are using Electron From 61e7d4f807e791ad147cb6879fb0eb947425db41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 10 May 2021 18:22:06 +0200 Subject: [PATCH 36/53] Remove some leftovers from placeScreensharingCall() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 07cf80ff5de..ecd4e167e95 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1001,32 +1001,13 @@ export class MatrixCall extends EventEmitter { this.stopAllMedia(); return; } - this.localAVStream = stream; - logger.info("Got local AV stream with id " + this.localAVStream.id); + this.localAVStream = stream; + this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); this.setState(CallState.CreateOffer); + logger.info("Got local AV stream with id " + this.localAVStream.id); logger.debug("gotUserMediaForInvite -> " + this.type); - - if (this.screenSharingStream) { - logger.debug( - "Setting screen sharing stream to the local video element", - ); - - this.pushLocalFeed(this.screenSharingStream, SDPStreamMetadataPurpose.Screenshare, false); - this.pushLocalFeed(this.localAVStream, SDPStreamMetadataPurpose.Usermedia); - - const videoScreensharingTrack = this.screenSharingStream.getTracks().find((track) => { - return track.kind === "video"; - }); - const userMediaVideoSender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; - }); - userMediaVideoSender.replaceTrack(videoScreensharingTrack); - } else { - this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); - } - // Now we wait for the negotiationneeded event }; From 582aafa552183b225009a38667f8c0bbfb61b6a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 10 May 2021 19:12:33 +0200 Subject: [PATCH 37/53] Simpler naming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index ecd4e167e95..d2d4b912bd0 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -833,13 +833,13 @@ export class MatrixCall extends EventEmitter { this.screenSharingStream = await getScreensharingStream(selectDesktopCapturerSource); if (!this.screenSharingStream) return false; - const videoScreensharingTrack = this.screenSharingStream.getTracks().find((track) => { + const track = this.screenSharingStream.getTracks().find((track) => { return track.kind === "video"; }); - const userMediaVideoSender = this.usermediaSenders.find((sender) => { + const sender = this.usermediaSenders.find((sender) => { return sender.track?.kind === "video"; }); - userMediaVideoSender.replaceTrack(videoScreensharingTrack); + sender.replaceTrack(track); this.pushLocalFeed(this.screenSharingStream, SDPStreamMetadataPurpose.Screenshare, false); From 2596c25ccc7dc8bd477c40b0461643c3e8fbf401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 10 May 2021 19:17:07 +0200 Subject: [PATCH 38/53] Add missing semicolon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d2d4b912bd0..f9e1924de65 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1356,7 +1356,7 @@ export class MatrixCall extends EventEmitter { } catch (err) { logger.debug("Error setting local description!", err); this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return + return; } if (this.peerConn.iceGatheringState === 'gathering') { From 00a28e743d64383ad63f2abdf4121362452371a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 10 May 2021 19:18:39 +0200 Subject: [PATCH 39/53] Move track.stop() to deleteFeedByStream() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index f9e1924de65..a03572375ff 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -546,6 +546,10 @@ export class MatrixCall extends EventEmitter { this.feeds.splice(this.feeds.indexOf(feed), 1); this.emit(CallEvent.FeedsChanged, this.feeds); + + for (const track of stream.getTracks()) { + track.stop(); + } } // The typescript definitions have this type as 'any' :( @@ -807,9 +811,6 @@ export class MatrixCall extends EventEmitter { for (const sender of this.screensharingSenders) { this.peerConn.removeTrack(sender); } - for (const track of this.screenSharingStream.getTracks()) { - track.stop(); - } this.deleteFeedByStream(this.screenSharingStream); this.screenSharingStream = null; return false; @@ -859,9 +860,6 @@ export class MatrixCall extends EventEmitter { }); sender.replaceTrack(track); - for (const track of this.screenSharingStream.getTracks()) { - track.stop(); - } this.deleteFeedByStream(this.screenSharingStream); this.screenSharingStream = null; From 5a8e5a9785a3f21b5684f91ba6c6308ae2b22bb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 13 May 2021 13:43:44 +0200 Subject: [PATCH 40/53] Log local SDPStreamMetadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index a03572375ff..9f8b035c0c4 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -429,6 +429,7 @@ export class MatrixCall extends EventEmitter { purpose: localFeed.purpose, }; } + logger.debug("Got local SDPStreamMetadata", metadata); return metadata; } From 44fc820f99044a0d1f3527fd7466bb4847226945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 13 May 2021 18:12:48 +0200 Subject: [PATCH 41/53] Make feed pushing methods more verbose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 9f8b035c0c4..10f1d7c35f0 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -467,7 +467,7 @@ export class MatrixCall extends EventEmitter { this.emit(CallEvent.FeedsChanged, this.feeds); } - logger.info(`Pushed remote stream with id ${stream.id}. Stream active? ${stream.active}`); + logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); } /** @@ -498,7 +498,7 @@ export class MatrixCall extends EventEmitter { this.emit(CallEvent.FeedsChanged, this.feeds); } - logger.info(`Pushed remote stream with id ${stream.id}. Stream active? ${stream.active}`); + logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); } private pushLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true) { @@ -522,13 +522,21 @@ export class MatrixCall extends EventEmitter { // Empty the array senderArray.splice(0, senderArray.length); + this.emit(CallEvent.FeedsChanged, this.feeds); for (const track of stream.getTracks()) { - logger.info(`Adding track with id ${track.id} and with kind ${track.kind} to peer connection`) + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${purpose}"` + + `) to peer connection`, + ) senderArray.push(this.peerConn.addTrack(track, stream)); } } - logger.info(`Pushed local stream with id ${stream.id}. Stream active? ${stream.active}`); + logger.info(`Pushed local stream (id="${stream.id}", active="${stream.active}", purpose="${purpose}")`); } private deleteAllFeeds() { From 5de189bfa3c97a0e66d107d51312f6bbac4e5773 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 15 May 2021 11:06:04 +0200 Subject: [PATCH 42/53] Improve logging in pushRemoteFeed() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 10f1d7c35f0..fee073eb27a 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -450,24 +450,24 @@ export class MatrixCall extends EventEmitter { } const userId = this.getOpponentMember().userId; - const streamMetadata = this.remoteSDPStreamMetadata[stream.id]; + const purpose = this.remoteSDPStreamMetadata[stream.id].purpose; - if (!streamMetadata) { + if (!purpose) { logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); return; } // Try to find a feed with the same purpose as the new stream, // if we find it replace the old stream with the new one - const existingFeed = this.getRemoteFeeds().find((feed) => feed.purpose === streamMetadata.purpose); + const existingFeed = this.getRemoteFeeds().find((feed) => feed.purpose === purpose); if (existingFeed) { existingFeed.setNewStream(stream); } else { - this.feeds.push(new CallFeed(stream, userId, streamMetadata.purpose, this.client, this.roomId)); + this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId)); this.emit(CallEvent.FeedsChanged, this.feeds); } - logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); + logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}", purpose=${purpose})`); } /** From 92e89ffbcf59992a62e963c2af0621f3c1b863f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 19 May 2021 21:06:35 +0200 Subject: [PATCH 43/53] Add getRidOfRTXCodecs() method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index fef2869bbb7..4b16972ceae 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1505,6 +1505,44 @@ export class MatrixCall extends EventEmitter { stream.addEventListener("removetrack", () => this.deleteFeedByStream(stream)); }; + /** + * This method removes all video/rtx codecs from video transceivers. + * This is necessary since they can cause problems. Without this the + * following steps should produce an error: + * Chromium calls Firefox + * Firefox answers + * Firefox starts screen-sharing + * Chromium starts screen-sharing + * Call crashes for Chromium with: + * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list. + * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs. + * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER) + */ + private getRidOfRTXCodecs() { + // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF + if (RTCRtpReceiver.getCapabilities && RTCRtpSender.getCapabilities) { + const recvCodecs = RTCRtpReceiver.getCapabilities("video").codecs; + const sendCodecs = RTCRtpSender.getCapabilities("video").codecs; + const codecs = [...sendCodecs, ...recvCodecs]; + + for (const codec of codecs) { + if (codec.mimeType === "video/rtx") { + const rtxCodecIndex = codecs.indexOf(codec); + codecs.splice(rtxCodecIndex, 1); + } + } + + for (const trans of this.peerConn.getTransceivers()) { + if ( + trans.sender.track?.kind === "video" || + trans.receiver.track?.kind === "video" + ) { + trans.setCodecPreferences(codecs); + } + } + } + } + onNegotiationNeeded = async () => { logger.info("Negotation is needed!"); @@ -1515,6 +1553,7 @@ export class MatrixCall extends EventEmitter { this.makingOffer = true; try { + this.getRidOfRTXCodecs(); const myOffer = await this.peerConn.createOffer(); await this.gotLocalOffer(myOffer); } catch (e) { From b1459a43ef7d74cbffbd92db4a0ce360bde0730f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 20 May 2021 15:11:23 +0200 Subject: [PATCH 44/53] Also put getRidOfRTXCodecs() before createAnswer() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just to be sure Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 4b16972ceae..5be011e4748 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1080,6 +1080,7 @@ export class MatrixCall extends EventEmitter { let myAnswer; try { + this.getRidOfRTXCodecs(); myAnswer = await this.peerConn.createAnswer(); } catch (err) { logger.debug("Failed to create answer: ", err); @@ -1306,6 +1307,7 @@ export class MatrixCall extends EventEmitter { for (const tranceiver of this.peerConn.getTransceivers()) { tranceiver.direction = this.isRemoteOnHold() ? 'inactive' : 'sendrecv'; } + this.getRidOfRTXCodecs(); const localDescription = await this.peerConn.createAnswer(); await this.peerConn.setLocalDescription(localDescription); // Now we've got our answer, set the direction to the outcome of the negotiation. From a1a5d85979bd46ee5656555703e9fed57f4c248a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 20 May 2021 15:49:56 +0200 Subject: [PATCH 45/53] Stop tracks only if disabling screen-sharing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The other thing lead to usermedia tracks being stopped when on hold Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 5be011e4748..af1a46c6c81 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -555,10 +555,6 @@ export class MatrixCall extends EventEmitter { this.feeds.splice(this.feeds.indexOf(feed), 1); this.emit(CallEvent.FeedsChanged, this.feeds); - - for (const track of stream.getTracks()) { - track.stop(); - } } // The typescript definitions have this type as 'any' :( @@ -821,6 +817,9 @@ export class MatrixCall extends EventEmitter { this.peerConn.removeTrack(sender); } this.deleteFeedByStream(this.screenSharingStream); + for (const track of this.screensharingSenders.getTracks()) { + track.stop(); + } this.screenSharingStream = null; return false; } @@ -870,6 +869,9 @@ export class MatrixCall extends EventEmitter { sender.replaceTrack(track); this.deleteFeedByStream(this.screenSharingStream); + for (const track of this.screenSharingStream.getTracks()) { + track.stop(); + } this.screenSharingStream = null; return false; From db4c6af472437b2d469d4d119e8f91129ba30fcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 20 May 2021 15:54:08 +0200 Subject: [PATCH 46/53] Fix an odd mistake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 3be346ff0d9..a288142c83c 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -817,7 +817,7 @@ export class MatrixCall extends EventEmitter { this.peerConn.removeTrack(sender); } this.deleteFeedByStream(this.screenSharingStream); - for (const track of this.screensharingSenders.getTracks()) { + for (const track of this.screenSharingStream.getTracks()) { track.stop(); } this.screenSharingStream = null; From 43198b0425bfedc1eb14774214670013143d23f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 27 May 2021 18:41:48 +0200 Subject: [PATCH 47/53] Disable RTX only for screen-sharing transceivers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index aca13009660..f25af4242fa 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1509,9 +1509,9 @@ export class MatrixCall extends EventEmitter { }; /** - * This method removes all video/rtx codecs from video transceivers. - * This is necessary since they can cause problems. Without this the - * following steps should produce an error: + * This method removes all video/rtx codecs from screensharing video + * transceivers. This is necessary since they can cause problems. Without + * this the following steps should produce an error: * Chromium calls Firefox * Firefox answers * Firefox starts screen-sharing @@ -1537,8 +1537,11 @@ export class MatrixCall extends EventEmitter { for (const trans of this.peerConn.getTransceivers()) { if ( - trans.sender.track?.kind === "video" || - trans.receiver.track?.kind === "video" + this.screensharingSenders.includes(trans.sender) && + ( + trans.sender.track?.kind === "video" || + trans.receiver.track?.kind === "video" + ) ) { trans.setCodecPreferences(codecs); } From 75321220fd8b7a1cdd39d54db213b3ca4b6cb9f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 27 May 2021 18:49:02 +0200 Subject: [PATCH 48/53] Simplifie code - don't be an idiot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index f25af4242fa..821345cd483 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1523,28 +1523,28 @@ export class MatrixCall extends EventEmitter { */ private getRidOfRTXCodecs() { // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF - if (RTCRtpReceiver.getCapabilities && RTCRtpSender.getCapabilities) { - const recvCodecs = RTCRtpReceiver.getCapabilities("video").codecs; - const sendCodecs = RTCRtpSender.getCapabilities("video").codecs; - const codecs = [...sendCodecs, ...recvCodecs]; - - for (const codec of codecs) { - if (codec.mimeType === "video/rtx") { - const rtxCodecIndex = codecs.indexOf(codec); - codecs.splice(rtxCodecIndex, 1); - } + if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return; + + const recvCodecs = RTCRtpReceiver.getCapabilities("video").codecs; + const sendCodecs = RTCRtpSender.getCapabilities("video").codecs; + const codecs = [...sendCodecs, ...recvCodecs]; + + for (const codec of codecs) { + if (codec.mimeType === "video/rtx") { + const rtxCodecIndex = codecs.indexOf(codec); + codecs.splice(rtxCodecIndex, 1); } + } - for (const trans of this.peerConn.getTransceivers()) { - if ( - this.screensharingSenders.includes(trans.sender) && + for (const trans of this.peerConn.getTransceivers()) { + if ( + this.screensharingSenders.includes(trans.sender) && ( trans.sender.track?.kind === "video" || trans.receiver.track?.kind === "video" ) - ) { - trans.setCodecPreferences(codecs); - } + ) { + trans.setCodecPreferences(codecs); } } } From 0a0489750c488bbdaad0552f8fac7920304adacb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 28 May 2021 17:37:55 +0200 Subject: [PATCH 49/53] Add missing space MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/call.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 885df977f06..8b2a27a2772 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { TestClient} from '../../TestClient'; +import { TestClient } from '../../TestClient'; import { MatrixCall, CallErrorCode, CallEvent } from '../../../src/webrtc/call'; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes'; From 5724462c2cd82b1da9225cd5aa0a62992ee8f090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 28 May 2021 17:45:52 +0200 Subject: [PATCH 50/53] Delint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/call.spec.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 8b2a27a2772..cf46053d208 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -305,7 +305,7 @@ describe('Call', function() { await client.httpBackend.flush(); await callPromise; - call.getOpponentMember = () => {return {userId: "@bob:bar.uk"}}; + call.getOpponentMember = () => {return { userId: "@bob:bar.uk" }}; await call.onAnswerReceived({ getContent: () => { @@ -319,14 +319,16 @@ describe('Call', function() { [SDPStreamMetadataKey]: { "stream_id": { purpose: SDPStreamMetadataPurpose.Usermedia, - } - } + }, + }, }; }, }); - call.pushRemoteFeed({id: "stream_id"}); - expect(call.getFeeds().find((feed) => feed.stream.id === "stream_id")?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia); + call.pushRemoteFeed({ id: "stream_id" }); + expect(call.getFeeds().find((feed) => { + return feed.stream.id === "stream_id"; + })?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia); }); it("should fallback to replaceTrack() if the other side doesn't support SPDStreamMetadata", async () => { @@ -334,7 +336,7 @@ describe('Call', function() { await client.httpBackend.flush(); await callPromise; - call.getOpponentMember = () => {return {userId: "@bob:bar.uk"}}; + call.getOpponentMember = () => {return { userId: "@bob:bar.uk" }}; await call.onAnswerReceived({ getContent: () => { From 0e05f9fd734bb63627cd186fd5dde029938d036d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 4 Jun 2021 11:41:35 +0200 Subject: [PATCH 51/53] Styling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- spec/unit/webrtc/call.spec.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index cf46053d208..69a3553b01c 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -305,7 +305,9 @@ describe('Call', function() { await client.httpBackend.flush(); await callPromise; - call.getOpponentMember = () => {return { userId: "@bob:bar.uk" }}; + call.getOpponentMember = () => { + return { userId: "@bob:bar.uk" }; + }; await call.onAnswerReceived({ getContent: () => { @@ -336,7 +338,9 @@ describe('Call', function() { await client.httpBackend.flush(); await callPromise; - call.getOpponentMember = () => {return { userId: "@bob:bar.uk" }}; + call.getOpponentMember = () => { + return { userId: "@bob:bar.uk" }; + }; await call.onAnswerReceived({ getContent: () => { From c1625e5c27d78de9fd15760273e008615bf60042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 4 Jun 2021 11:44:31 +0200 Subject: [PATCH 52/53] More styling :( MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 72e6fde37f7..0e37dac462c 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -481,7 +481,7 @@ export class MatrixCall extends EventEmitter { const userId = this.getOpponentMember().userId; // We can guess the purpose here since the other client can only send one stream const purpose = SDPStreamMetadataPurpose.Usermedia; - const oldRemoteStream = this.feeds.find((feed) => {return !feed.isLocal()})?.stream; + const oldRemoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; // Note that we check by ID and always set the remote stream: Chrome appears // to make new stream objects when transceiver directionality is changed and the 'active' @@ -535,7 +535,7 @@ export class MatrixCall extends EventEmitter { `streamId="${stream.id}", ` + `streamPurpose="${purpose}"` + `) to peer connection`, - ) + ); senderArray.push(this.peerConn.addTrack(track, stream)); } } @@ -1299,7 +1299,7 @@ export class MatrixCall extends EventEmitter { if (metadata) { this.remoteSDPStreamMetadata = metadata; } else { - logger.warn("Received negotiation event without SDPStreamMetadata!") + logger.warn("Received negotiation event without SDPStreamMetadata!"); } try { From fcbbcc0398e0942b5d0816fcd6a88bfdc3e4ac25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 22 Jul 2021 09:24:59 +0200 Subject: [PATCH 53/53] Always return true for voice calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index de81d47c90a..6a9a8be7667 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -901,6 +901,7 @@ export class MatrixCall extends EventEmitter { * (including if the call is not set up yet). */ isLocalVideoMuted(): boolean { + if (this.type === CallType.Voice) return true; return this.vidMuted; }