From 47db8f23cb71f3cadf55f6c391daa8c7bf779efb Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 18 Jul 2023 14:30:59 +0100 Subject: [PATCH 01/12] Spec for MSC3077 Specs https://github.com/matrix-org/matrix-spec-proposals/pull/3077 --- .../client-server-api/modules/voip_events.md | 40 +++++++++++++------ data/event-schemas/schema/m.call.answer.yaml | 17 ++++++++ data/event-schemas/schema/m.call.invite.yaml | 17 ++++++++ .../schema/m.call.negotiate.yaml | 17 ++++++++ 4 files changed, 79 insertions(+), 12 deletions(-) diff --git a/content/client-server-api/modules/voip_events.md b/content/client-server-api/modules/voip_events.md index 102e3dcd7..415b8a05c 100644 --- a/content/client-server-api/modules/voip_events.md +++ b/content/client-server-api/modules/voip_events.md @@ -171,18 +171,34 @@ In response to an incoming invite, a client may do one of several things: ##### Streams -Clients are expected to send one stream with one track of kind `audio` (creating a -voice call). They can optionally send a second track in the same stream of kind -`video` (creating a video call). - -Clients implementing this specification use the first stream and will ignore -any streamless tracks. Note that in the JavaScript WebRTC API, this means -`addTrack()` must be passed two parameters: a track and a stream, not just a -track, and in a video call the stream must be the same for both audio and video -track. - -A client may send other streams and tracks but the behaviour of the other party -with respect to presenting such streams and tracks is undefined. +Clients may send more than one stream in a VoIP call. Metadata can be included in +the `m.call.invite`, `m.call.answer` and `m.call.negotiate` events to describe the +streams being sent. This is the `sdp_stream_metadata` field. + +This field is an object in which each key is one stream `id` in the session +description. The values are objects with the following fields: + + * `purpose` - a string indicating the purpose of the stream. For compatibility + between client the following values are defined: + * `m.usermedia` - stream that contains the webcam and/or microphone tracks + * `m.screenshare` - stream with the screen-sharing tracks + +If an incoming stream is not described in `sdp_stream_metadata` and +`sdp_stream_metadata` is present, the stream should be ignored. If a stream has +a `purpose` of an unknown type (i.e. not `m.usermedia` or `m.screenshare`), it +should be ignored. + +Clients implementing this specification will ignore any streamless tracks. Note +that in the JavaScript WebRTC API, this means `addTrack()` must be passed two +parameters: a track and a stream, not just a track, and in a video call the +stream must be the same for both audio and video track. + +During the initial invite and answer exchange clients find out if the field +`sdp_stream_metadata` is missing. For backwards compatibility if it is not +present in the event sent by the opponent, the client should ignore any new +incoming streams (i.e. it should use the first one) and it shouldn't send more +than one stream (i.e. clients cannot send a video feed and a screenshare at the +same time, as is the case in current clients). ##### Invitees The `invitee` field should be added whenever the call is intended for one diff --git a/data/event-schemas/schema/m.call.answer.yaml b/data/event-schemas/schema/m.call.answer.yaml index 163690be2..44e40dd21 100644 --- a/data/event-schemas/schema/m.call.answer.yaml +++ b/data/event-schemas/schema/m.call.answer.yaml @@ -27,6 +27,23 @@ } }, "required": ["type", "sdp"] + }, + "sdp_stream_metadata": { + "type": "object", + "title": "SDP Stream Metadata", + "x-addedInMatrixVersion": "1.7", + "description": "Object describing the streams that will be sent", + "additionalProperties": { + "type": "object", + "title": "Stream Metadata Block", + "properties": { + "purpose": { + "type": "string", + "enum": ["m.usermedia", "m.screenshare"] + } + }, + "required": ["purpose"] + } } }, "required": ["answer"] diff --git a/data/event-schemas/schema/m.call.invite.yaml b/data/event-schemas/schema/m.call.invite.yaml index 72020b266..8bdde9b85 100644 --- a/data/event-schemas/schema/m.call.invite.yaml +++ b/data/event-schemas/schema/m.call.invite.yaml @@ -35,7 +35,24 @@ "invitee": { "type": "string", "description": "The ID of the user being called. If omitted, any user in the room can answer.", + "x-addedInMatrixVersion": "1.7" + }, + "sdp_stream_metadata": { + "type": "object", + "title": "SDP Stream Metadata", "x-addedInMatrixVersion": "1.7", + "description": "Object describing the streams that will be sent", + "additionalProperties": { + "type": "object", + "title": "Stream Metadata Block", + "properties": { + "purpose": { + "type": "string", + "enum": ["m.usermedia", "m.screenshare"] + } + }, + "required": ["purpose"] + } } }, "required": ["offer", "lifetime"] diff --git a/data/event-schemas/schema/m.call.negotiate.yaml b/data/event-schemas/schema/m.call.negotiate.yaml index abc5ef1d8..f34d6502e 100644 --- a/data/event-schemas/schema/m.call.negotiate.yaml +++ b/data/event-schemas/schema/m.call.negotiate.yaml @@ -64,6 +64,23 @@ properties: Once the invite age exceeds this value, clients should discard it. They should also no longer show the call as awaiting an answer in the UI. + sdp_stream_metadata: + type: object + title: SDP Stream Metadata + x-addedInMatrixVersion: '1.7' + description: Object describing the streams that will be sent + additionalProperties: + type: object + title: Stream Metadata Block + properties: + purpose: + type: string + enum: + - m.usermedia + - m.screenshare + required: + - purpose + required: - offer - lifetime From fcd18bb33088bcf93125f7432fc846af6ee7b2c6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 18 Jul 2023 14:33:40 +0100 Subject: [PATCH 02/12] Newsfragment --- changelogs/client_server/newsfragments/1602.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1602.feature diff --git a/changelogs/client_server/newsfragments/1602.feature b/changelogs/client_server/newsfragments/1602.feature new file mode 100644 index 000000000..d6d7bfa9d --- /dev/null +++ b/changelogs/client_server/newsfragments/1602.feature @@ -0,0 +1 @@ +Add specification for [MSC3077: Multi-stream VoIP](https://github.com/matrix-org/matrix-spec-proposals/pull/3077]. From 3cfade3cdc447dbde0b9e80c6754a73da7c4a978 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 18 Jul 2023 14:35:38 +0100 Subject: [PATCH 03/12] Add examples --- data/event-schemas/examples/m.call.answer.yaml | 10 +++++++++- data/event-schemas/examples/m.call.invite.yaml | 10 +++++++++- data/event-schemas/examples/m.call.negotiate.yaml | 10 +++++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/data/event-schemas/examples/m.call.answer.yaml b/data/event-schemas/examples/m.call.answer.yaml index 78b488783..ba2cc5d25 100644 --- a/data/event-schemas/examples/m.call.answer.yaml +++ b/data/event-schemas/examples/m.call.answer.yaml @@ -8,6 +8,14 @@ "answer": { "type" : "answer", "sdp" : "v=0\r\no=- 6584580628695956864 2 IN IP4 127.0.0.1[...]" - } + }, + "sdp_stream_metadata": { + "271828182845": { + "purpose": "m.screenshare", + }, + "314159265358": { + "purpose": "m.usermedia", + }, + }, } } diff --git a/data/event-schemas/examples/m.call.invite.yaml b/data/event-schemas/examples/m.call.invite.yaml index 45600001e..4ed440f9f 100644 --- a/data/event-schemas/examples/m.call.invite.yaml +++ b/data/event-schemas/examples/m.call.invite.yaml @@ -9,6 +9,14 @@ "offer": { "type" : "offer", "sdp" : "v=0\r\no=- 6584580628695956864 2 IN IP4 127.0.0.1[...]" - } + }, + "sdp_stream_metadata": { + "271828182845": { + "purpose": "m.screenshare", + }, + "314159265358": { + "purpose": "m.usermedia", + }, + }, } } diff --git a/data/event-schemas/examples/m.call.negotiate.yaml b/data/event-schemas/examples/m.call.negotiate.yaml index f4ad8587e..32bb652d3 100644 --- a/data/event-schemas/examples/m.call.negotiate.yaml +++ b/data/event-schemas/examples/m.call.negotiate.yaml @@ -9,6 +9,14 @@ "offer": { "type" : "offer", "sdp" : "v=0\r\no=- 6584580628695956864 2 IN IP4 127.0.0.1[...]" - } + }, + "sdp_stream_metadata": { + "271828182845": { + "purpose": "m.screenshare", + }, + "314159265358": { + "purpose": "m.usermedia", + }, + }, } } From ffac074f3d283fdc4877dcca9ace5c9110272800 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 18 Jul 2023 14:44:57 +0100 Subject: [PATCH 04/12] Maybe make example validator happy? --- data/event-schemas/examples/m.call.answer.yaml | 2 +- data/event-schemas/examples/m.call.invite.yaml | 2 +- data/event-schemas/examples/m.call.negotiate.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data/event-schemas/examples/m.call.answer.yaml b/data/event-schemas/examples/m.call.answer.yaml index ba2cc5d25..b3eb1be42 100644 --- a/data/event-schemas/examples/m.call.answer.yaml +++ b/data/event-schemas/examples/m.call.answer.yaml @@ -16,6 +16,6 @@ "314159265358": { "purpose": "m.usermedia", }, - }, + } } } diff --git a/data/event-schemas/examples/m.call.invite.yaml b/data/event-schemas/examples/m.call.invite.yaml index 4ed440f9f..9958cfe7d 100644 --- a/data/event-schemas/examples/m.call.invite.yaml +++ b/data/event-schemas/examples/m.call.invite.yaml @@ -17,6 +17,6 @@ "314159265358": { "purpose": "m.usermedia", }, - }, + } } } diff --git a/data/event-schemas/examples/m.call.negotiate.yaml b/data/event-schemas/examples/m.call.negotiate.yaml index 32bb652d3..fc79321f2 100644 --- a/data/event-schemas/examples/m.call.negotiate.yaml +++ b/data/event-schemas/examples/m.call.negotiate.yaml @@ -17,6 +17,6 @@ "314159265358": { "purpose": "m.usermedia", }, - }, + } } } From 4e41985f9d806eea274aab40d9a371723de77b28 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 18 Jul 2023 15:20:53 +0100 Subject: [PATCH 05/12] Fix example JSON --- data/event-schemas/examples/m.call.answer.yaml | 6 +++--- data/event-schemas/examples/m.call.invite.yaml | 6 +++--- data/event-schemas/examples/m.call.negotiate.yaml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/data/event-schemas/examples/m.call.answer.yaml b/data/event-schemas/examples/m.call.answer.yaml index b3eb1be42..8a6273603 100644 --- a/data/event-schemas/examples/m.call.answer.yaml +++ b/data/event-schemas/examples/m.call.answer.yaml @@ -11,11 +11,11 @@ }, "sdp_stream_metadata": { "271828182845": { - "purpose": "m.screenshare", + "purpose": "m.screenshare" }, "314159265358": { - "purpose": "m.usermedia", - }, + "purpose": "m.usermedia" + } } } } diff --git a/data/event-schemas/examples/m.call.invite.yaml b/data/event-schemas/examples/m.call.invite.yaml index 9958cfe7d..9547854b5 100644 --- a/data/event-schemas/examples/m.call.invite.yaml +++ b/data/event-schemas/examples/m.call.invite.yaml @@ -12,11 +12,11 @@ }, "sdp_stream_metadata": { "271828182845": { - "purpose": "m.screenshare", + "purpose": "m.screenshare" }, "314159265358": { - "purpose": "m.usermedia", - }, + "purpose": "m.usermedia" + } } } } diff --git a/data/event-schemas/examples/m.call.negotiate.yaml b/data/event-schemas/examples/m.call.negotiate.yaml index fc79321f2..3120ee543 100644 --- a/data/event-schemas/examples/m.call.negotiate.yaml +++ b/data/event-schemas/examples/m.call.negotiate.yaml @@ -12,11 +12,11 @@ }, "sdp_stream_metadata": { "271828182845": { - "purpose": "m.screenshare", + "purpose": "m.screenshare" }, "314159265358": { - "purpose": "m.usermedia", - }, + "purpose": "m.usermedia" + } } } } From 8ce2b946abce0a948220690ce8578cbee60ea0e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Wed, 28 Feb 2024 21:47:51 +0100 Subject: [PATCH 06/12] Apply suggestions on VoIP module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kévin Commaille --- .../client-server-api/modules/voip_events.md | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/content/client-server-api/modules/voip_events.md b/content/client-server-api/modules/voip_events.md index 415b8a05c..1e4c3db7b 100644 --- a/content/client-server-api/modules/voip_events.md +++ b/content/client-server-api/modules/voip_events.md @@ -172,16 +172,17 @@ In response to an incoming invite, a client may do one of several things: ##### Streams Clients may send more than one stream in a VoIP call. Metadata can be included in -the `m.call.invite`, `m.call.answer` and `m.call.negotiate` events to describe the -streams being sent. This is the `sdp_stream_metadata` field. +the [`m.call.invite`](/client-server-api/#mcallinvite), [`m.call.answer`](/client-server-api/#mcallanswer) +and [`m.call.negotiate`](/client-server-api/#mcallnegotiate) events to describe +the streams being sent, using the `sdp_stream_metadata` property. -This field is an object in which each key is one stream `id` in the session -description. The values are objects with the following fields: +`sdp_stream_metadata` maps from the `id` of a stream in the session description, +to metadata about that stream. Currently only one property is defined for the +metadata. This is `purpose`, which should be a string indicating the purpose of +the stream. The following `purpose`s are defined: - * `purpose` - a string indicating the purpose of the stream. For compatibility - between client the following values are defined: - * `m.usermedia` - stream that contains the webcam and/or microphone tracks - * `m.screenshare` - stream with the screen-sharing tracks +* `m.usermedia` - stream that contains the webcam and/or microphone tracks +* `m.screenshare` - stream with the screen-sharing tracks If an incoming stream is not described in `sdp_stream_metadata` and `sdp_stream_metadata` is present, the stream should be ignored. If a stream has @@ -193,12 +194,11 @@ that in the JavaScript WebRTC API, this means `addTrack()` must be passed two parameters: a track and a stream, not just a track, and in a video call the stream must be the same for both audio and video track. -During the initial invite and answer exchange clients find out if the field -`sdp_stream_metadata` is missing. For backwards compatibility if it is not -present in the event sent by the opponent, the client should ignore any new -incoming streams (i.e. it should use the first one) and it shouldn't send more -than one stream (i.e. clients cannot send a video feed and a screenshare at the -same time, as is the case in current clients). +For backwards compatibility, if `sdp_stream_metadata` is not present in the +initial [`m.call.invite`](/client-server-api/#mcallinvite) or [`m.call.answer`](/client-server-api/#mcallanswer) +event sent by the other party, the client should ignore any new incoming streams +(i.e. it should use the first one) and it shouldn't send more than one stream +(i.e. clients cannot send a video feed and a screenshare at the same time). ##### Invitees The `invitee` field should be added whenever the call is intended for one From ef9546dabcab90f7e34ed89c9c638035b010296b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Wed, 28 Feb 2024 22:07:43 +0100 Subject: [PATCH 07/12] Factor out sdp_stream_metadata definition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kévin Commaille --- .../sdp_stream_metadata.yaml | 27 +++++++++++++++++++ data/event-schemas/schema/m.call.answer.yaml | 16 +---------- data/event-schemas/schema/m.call.invite.yaml | 16 +---------- .../schema/m.call.negotiate.yaml | 17 +----------- 4 files changed, 30 insertions(+), 46 deletions(-) create mode 100644 data/event-schemas/schema/core-event-schema/sdp_stream_metadata.yaml diff --git a/data/event-schemas/schema/core-event-schema/sdp_stream_metadata.yaml b/data/event-schemas/schema/core-event-schema/sdp_stream_metadata.yaml new file mode 100644 index 000000000..6d06f8bce --- /dev/null +++ b/data/event-schemas/schema/core-event-schema/sdp_stream_metadata.yaml @@ -0,0 +1,27 @@ +type: object +x-addedInMatrixVersion: "1.10" +description: |- + Metadata describing the [streams](/client-server-api/#streams) that will be + sent. + + This is a map of stream ID to metadata about the stream. +additionalProperties: + type: object + title: StreamMetadata + description: Metadata describing a stream. + properties: + purpose: + type: string + enum: + - m.usermedia + - m.screenshare + description: |- + The purpose of the stream. + + The possible values are: + + * `m.usermedia` - stream that contains the webcam and/or microphone + tracks + * `m.screenshare` - stream with the screen-sharing tracks + required: + - purpose diff --git a/data/event-schemas/schema/m.call.answer.yaml b/data/event-schemas/schema/m.call.answer.yaml index 44e40dd21..987b78f87 100644 --- a/data/event-schemas/schema/m.call.answer.yaml +++ b/data/event-schemas/schema/m.call.answer.yaml @@ -29,21 +29,7 @@ "required": ["type", "sdp"] }, "sdp_stream_metadata": { - "type": "object", - "title": "SDP Stream Metadata", - "x-addedInMatrixVersion": "1.7", - "description": "Object describing the streams that will be sent", - "additionalProperties": { - "type": "object", - "title": "Stream Metadata Block", - "properties": { - "purpose": { - "type": "string", - "enum": ["m.usermedia", "m.screenshare"] - } - }, - "required": ["purpose"] - } + "$ref": "core-event-schema/sdp_stream_metadata.yaml" } }, "required": ["answer"] diff --git a/data/event-schemas/schema/m.call.invite.yaml b/data/event-schemas/schema/m.call.invite.yaml index 8bdde9b85..d779d99bd 100644 --- a/data/event-schemas/schema/m.call.invite.yaml +++ b/data/event-schemas/schema/m.call.invite.yaml @@ -38,21 +38,7 @@ "x-addedInMatrixVersion": "1.7" }, "sdp_stream_metadata": { - "type": "object", - "title": "SDP Stream Metadata", - "x-addedInMatrixVersion": "1.7", - "description": "Object describing the streams that will be sent", - "additionalProperties": { - "type": "object", - "title": "Stream Metadata Block", - "properties": { - "purpose": { - "type": "string", - "enum": ["m.usermedia", "m.screenshare"] - } - }, - "required": ["purpose"] - } + "$ref": "core-event-schema/sdp_stream_metadata.yaml" } }, "required": ["offer", "lifetime"] diff --git a/data/event-schemas/schema/m.call.negotiate.yaml b/data/event-schemas/schema/m.call.negotiate.yaml index 2fb4b0aad..16674efdc 100644 --- a/data/event-schemas/schema/m.call.negotiate.yaml +++ b/data/event-schemas/schema/m.call.negotiate.yaml @@ -64,22 +64,7 @@ properties: description: The time in milliseconds that the negotiation is valid for. Once the negotiation age exceeds this value, clients should discard it. sdp_stream_metadata: - type: object - title: SDP Stream Metadata - x-addedInMatrixVersion: '1.7' - description: Object describing the streams that will be sent - additionalProperties: - type: object - title: Stream Metadata Block - properties: - purpose: - type: string - enum: - - m.usermedia - - m.screenshare - required: - - purpose - + $ref: core-event-schema/sdp_stream_metadata.yaml required: - description - lifetime From d6d3c50729f1f6e31f3fc560a091875dffcb2c14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Wed, 28 Feb 2024 22:21:20 +0100 Subject: [PATCH 08/12] Update changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kévin Commaille --- changelogs/client_server/newsfragments/1602.feature | 1 - changelogs/client_server/newsfragments/1735.feature | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 changelogs/client_server/newsfragments/1602.feature create mode 100644 changelogs/client_server/newsfragments/1735.feature diff --git a/changelogs/client_server/newsfragments/1602.feature b/changelogs/client_server/newsfragments/1602.feature deleted file mode 100644 index d6d7bfa9d..000000000 --- a/changelogs/client_server/newsfragments/1602.feature +++ /dev/null @@ -1 +0,0 @@ -Add specification for [MSC3077: Multi-stream VoIP](https://github.com/matrix-org/matrix-spec-proposals/pull/3077]. diff --git a/changelogs/client_server/newsfragments/1735.feature b/changelogs/client_server/newsfragments/1735.feature new file mode 100644 index 000000000..1d7641421 --- /dev/null +++ b/changelogs/client_server/newsfragments/1735.feature @@ -0,0 +1 @@ +Add support for multi-stream VoIP, as per [MSC3077](https://github.com/matrix-org/matrix-spec-proposals/pull/3077). \ No newline at end of file From 671685e33009b174f668c5194c0fa33e07dab0e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Wed, 13 Mar 2024 10:32:28 +0100 Subject: [PATCH 09/12] Rework paragraph about streamless tracks and move to the end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kévin Commaille --- content/client-server-api/modules/voip_events.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/content/client-server-api/modules/voip_events.md b/content/client-server-api/modules/voip_events.md index 1e4c3db7b..a95f94f5e 100644 --- a/content/client-server-api/modules/voip_events.md +++ b/content/client-server-api/modules/voip_events.md @@ -189,17 +189,14 @@ If an incoming stream is not described in `sdp_stream_metadata` and a `purpose` of an unknown type (i.e. not `m.usermedia` or `m.screenshare`), it should be ignored. -Clients implementing this specification will ignore any streamless tracks. Note -that in the JavaScript WebRTC API, this means `addTrack()` must be passed two -parameters: a track and a stream, not just a track, and in a video call the -stream must be the same for both audio and video track. - For backwards compatibility, if `sdp_stream_metadata` is not present in the initial [`m.call.invite`](/client-server-api/#mcallinvite) or [`m.call.answer`](/client-server-api/#mcallanswer) event sent by the other party, the client should ignore any new incoming streams (i.e. it should use the first one) and it shouldn't send more than one stream (i.e. clients cannot send a video feed and a screenshare at the same time). +Clients implementing this specification should ignore any streamless tracks. + ##### Invitees The `invitee` field should be added whenever the call is intended for one specific user, and should be set to the Matrix user ID of that user. Invites From 02706b56cc34bd7e5e12d836d5faa8e418dfcd84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Wed, 13 Mar 2024 10:50:35 +0100 Subject: [PATCH 10/12] Rework paragraph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kévin Commaille --- .../client-server-api/modules/voip_events.md | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/content/client-server-api/modules/voip_events.md b/content/client-server-api/modules/voip_events.md index a95f94f5e..4df0ba13f 100644 --- a/content/client-server-api/modules/voip_events.md +++ b/content/client-server-api/modules/voip_events.md @@ -171,29 +171,29 @@ In response to an incoming invite, a client may do one of several things: ##### Streams -Clients may send more than one stream in a VoIP call. Metadata can be included in -the [`m.call.invite`](/client-server-api/#mcallinvite), [`m.call.answer`](/client-server-api/#mcallanswer) -and [`m.call.negotiate`](/client-server-api/#mcallnegotiate) events to describe -the streams being sent, using the `sdp_stream_metadata` property. +Clients may send more than one stream in a VoIP call. The streams should be +differentiated by including metadata in the [`m.call.invite`](/client-server-api/#mcallinvite), +[`m.call.answer`](/client-server-api/#mcallanswer) and [`m.call.negotiate`](/client-server-api/#mcallnegotiate) +events, using the `sdp_stream_metadata` property. `sdp_stream_metadata` maps from the `id` of a stream in the session description, -to metadata about that stream. Currently only one property is defined for the +to metadata about that stream. Currently only one property is defined for the metadata. This is `purpose`, which should be a string indicating the purpose of the stream. The following `purpose`s are defined: * `m.usermedia` - stream that contains the webcam and/or microphone tracks * `m.screenshare` - stream with the screen-sharing tracks -If an incoming stream is not described in `sdp_stream_metadata` and -`sdp_stream_metadata` is present, the stream should be ignored. If a stream has -a `purpose` of an unknown type (i.e. not `m.usermedia` or `m.screenshare`), it -should be ignored. +If `sdp_stream_metadata` is present and an incoming stream is not listed in it, +the stream should be ignored. If a stream has a `purpose` of an unknown type, it +should also be ignored. For backwards compatibility, if `sdp_stream_metadata` is not present in the initial [`m.call.invite`](/client-server-api/#mcallinvite) or [`m.call.answer`](/client-server-api/#mcallanswer) -event sent by the other party, the client should ignore any new incoming streams -(i.e. it should use the first one) and it shouldn't send more than one stream -(i.e. clients cannot send a video feed and a screenshare at the same time). +event sent by the other party, the client should assume that this property is +not supported by the other party. It means that multiple streams cannot be +differentiated: the client should only use the first incoming stream and +shouldn't send more than one stream. Clients implementing this specification should ignore any streamless tracks. From b0cfa292216ae87902851305e02226330315ca3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Wed, 13 Mar 2024 10:58:45 +0100 Subject: [PATCH 11/12] Move sdp_stream_metadata schema to new components folder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kévin Commaille --- .../{core-event-schema => components}/sdp_stream_metadata.yaml | 0 data/event-schemas/schema/m.call.answer.yaml | 2 +- data/event-schemas/schema/m.call.invite.yaml | 2 +- data/event-schemas/schema/m.call.negotiate.yaml | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename data/event-schemas/schema/{core-event-schema => components}/sdp_stream_metadata.yaml (100%) diff --git a/data/event-schemas/schema/core-event-schema/sdp_stream_metadata.yaml b/data/event-schemas/schema/components/sdp_stream_metadata.yaml similarity index 100% rename from data/event-schemas/schema/core-event-schema/sdp_stream_metadata.yaml rename to data/event-schemas/schema/components/sdp_stream_metadata.yaml diff --git a/data/event-schemas/schema/m.call.answer.yaml b/data/event-schemas/schema/m.call.answer.yaml index 987b78f87..15e072020 100644 --- a/data/event-schemas/schema/m.call.answer.yaml +++ b/data/event-schemas/schema/m.call.answer.yaml @@ -29,7 +29,7 @@ "required": ["type", "sdp"] }, "sdp_stream_metadata": { - "$ref": "core-event-schema/sdp_stream_metadata.yaml" + "$ref": "components/sdp_stream_metadata.yaml" } }, "required": ["answer"] diff --git a/data/event-schemas/schema/m.call.invite.yaml b/data/event-schemas/schema/m.call.invite.yaml index d779d99bd..c688d7b31 100644 --- a/data/event-schemas/schema/m.call.invite.yaml +++ b/data/event-schemas/schema/m.call.invite.yaml @@ -38,7 +38,7 @@ "x-addedInMatrixVersion": "1.7" }, "sdp_stream_metadata": { - "$ref": "core-event-schema/sdp_stream_metadata.yaml" + "$ref": "components/sdp_stream_metadata.yaml" } }, "required": ["offer", "lifetime"] diff --git a/data/event-schemas/schema/m.call.negotiate.yaml b/data/event-schemas/schema/m.call.negotiate.yaml index 16674efdc..e1a14f6f3 100644 --- a/data/event-schemas/schema/m.call.negotiate.yaml +++ b/data/event-schemas/schema/m.call.negotiate.yaml @@ -64,7 +64,7 @@ properties: description: The time in milliseconds that the negotiation is valid for. Once the negotiation age exceeds this value, clients should discard it. sdp_stream_metadata: - $ref: core-event-schema/sdp_stream_metadata.yaml + $ref: components/sdp_stream_metadata.yaml required: - description - lifetime From f8ff5812083d22b4d6db46f5fd59b427313e700e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Commaille?= Date: Wed, 13 Mar 2024 11:01:34 +0100 Subject: [PATCH 12/12] Apply review suggestion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kévin Commaille --- .../schema/components/sdp_stream_metadata.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/event-schemas/schema/components/sdp_stream_metadata.yaml b/data/event-schemas/schema/components/sdp_stream_metadata.yaml index 6d06f8bce..f16b4cbdc 100644 --- a/data/event-schemas/schema/components/sdp_stream_metadata.yaml +++ b/data/event-schemas/schema/components/sdp_stream_metadata.yaml @@ -20,8 +20,8 @@ additionalProperties: The possible values are: - * `m.usermedia` - stream that contains the webcam and/or microphone - tracks - * `m.screenshare` - stream with the screen-sharing tracks + * `m.usermedia`: Stream that contains the webcam and/or microphone + tracks. + * `m.screenshare`: Stream with the screen-sharing tracks. required: - purpose