From 47db8f23cb71f3cadf55f6c391daa8c7bf779efb Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 18 Jul 2023 14:30:59 +0100 Subject: [PATCH 1/5] Spec for MSC3077 Specs https://github.com/matrix-org/matrix-spec-proposals/pull/3077 --- .../client-server-api/modules/voip_events.md | 40 +++++++++++++------ data/event-schemas/schema/m.call.answer.yaml | 17 ++++++++ data/event-schemas/schema/m.call.invite.yaml | 17 ++++++++ .../schema/m.call.negotiate.yaml | 17 ++++++++ 4 files changed, 79 insertions(+), 12 deletions(-) diff --git a/content/client-server-api/modules/voip_events.md b/content/client-server-api/modules/voip_events.md index 102e3dcd7..415b8a05c 100644 --- a/content/client-server-api/modules/voip_events.md +++ b/content/client-server-api/modules/voip_events.md @@ -171,18 +171,34 @@ In response to an incoming invite, a client may do one of several things: ##### Streams -Clients are expected to send one stream with one track of kind `audio` (creating a -voice call). They can optionally send a second track in the same stream of kind -`video` (creating a video call). - -Clients implementing this specification use the first stream and will ignore -any streamless tracks. Note that in the JavaScript WebRTC API, this means -`addTrack()` must be passed two parameters: a track and a stream, not just a -track, and in a video call the stream must be the same for both audio and video -track. - -A client may send other streams and tracks but the behaviour of the other party -with respect to presenting such streams and tracks is undefined. +Clients may send more than one stream in a VoIP call. Metadata can be included in +the `m.call.invite`, `m.call.answer` and `m.call.negotiate` events to describe the +streams being sent. This is the `sdp_stream_metadata` field. + +This field is an object in which each key is one stream `id` in the session +description. The values are objects with the following fields: + + * `purpose` - a string indicating the purpose of the stream. For compatibility + between client the following values are defined: + * `m.usermedia` - stream that contains the webcam and/or microphone tracks + * `m.screenshare` - stream with the screen-sharing tracks + +If an incoming stream is not described in `sdp_stream_metadata` and +`sdp_stream_metadata` is present, the stream should be ignored. If a stream has +a `purpose` of an unknown type (i.e. not `m.usermedia` or `m.screenshare`), it +should be ignored. + +Clients implementing this specification will ignore any streamless tracks. Note +that in the JavaScript WebRTC API, this means `addTrack()` must be passed two +parameters: a track and a stream, not just a track, and in a video call the +stream must be the same for both audio and video track. + +During the initial invite and answer exchange clients find out if the field +`sdp_stream_metadata` is missing. For backwards compatibility if it is not +present in the event sent by the opponent, the client should ignore any new +incoming streams (i.e. it should use the first one) and it shouldn't send more +than one stream (i.e. clients cannot send a video feed and a screenshare at the +same time, as is the case in current clients). ##### Invitees The `invitee` field should be added whenever the call is intended for one diff --git a/data/event-schemas/schema/m.call.answer.yaml b/data/event-schemas/schema/m.call.answer.yaml index 163690be2..44e40dd21 100644 --- a/data/event-schemas/schema/m.call.answer.yaml +++ b/data/event-schemas/schema/m.call.answer.yaml @@ -27,6 +27,23 @@ } }, "required": ["type", "sdp"] + }, + "sdp_stream_metadata": { + "type": "object", + "title": "SDP Stream Metadata", + "x-addedInMatrixVersion": "1.7", + "description": "Object describing the streams that will be sent", + "additionalProperties": { + "type": "object", + "title": "Stream Metadata Block", + "properties": { + "purpose": { + "type": "string", + "enum": ["m.usermedia", "m.screenshare"] + } + }, + "required": ["purpose"] + } } }, "required": ["answer"] diff --git a/data/event-schemas/schema/m.call.invite.yaml b/data/event-schemas/schema/m.call.invite.yaml index 72020b266..8bdde9b85 100644 --- a/data/event-schemas/schema/m.call.invite.yaml +++ b/data/event-schemas/schema/m.call.invite.yaml @@ -35,7 +35,24 @@ "invitee": { "type": "string", "description": "The ID of the user being called. If omitted, any user in the room can answer.", + "x-addedInMatrixVersion": "1.7" + }, + "sdp_stream_metadata": { + "type": "object", + "title": "SDP Stream Metadata", "x-addedInMatrixVersion": "1.7", + "description": "Object describing the streams that will be sent", + "additionalProperties": { + "type": "object", + "title": "Stream Metadata Block", + "properties": { + "purpose": { + "type": "string", + "enum": ["m.usermedia", "m.screenshare"] + } + }, + "required": ["purpose"] + } } }, "required": ["offer", "lifetime"] diff --git a/data/event-schemas/schema/m.call.negotiate.yaml b/data/event-schemas/schema/m.call.negotiate.yaml index abc5ef1d8..f34d6502e 100644 --- a/data/event-schemas/schema/m.call.negotiate.yaml +++ b/data/event-schemas/schema/m.call.negotiate.yaml @@ -64,6 +64,23 @@ properties: Once the invite age exceeds this value, clients should discard it. They should also no longer show the call as awaiting an answer in the UI. + sdp_stream_metadata: + type: object + title: SDP Stream Metadata + x-addedInMatrixVersion: '1.7' + description: Object describing the streams that will be sent + additionalProperties: + type: object + title: Stream Metadata Block + properties: + purpose: + type: string + enum: + - m.usermedia + - m.screenshare + required: + - purpose + required: - offer - lifetime From fcd18bb33088bcf93125f7432fc846af6ee7b2c6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 18 Jul 2023 14:33:40 +0100 Subject: [PATCH 2/5] Newsfragment --- changelogs/client_server/newsfragments/1602.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/client_server/newsfragments/1602.feature diff --git a/changelogs/client_server/newsfragments/1602.feature b/changelogs/client_server/newsfragments/1602.feature new file mode 100644 index 000000000..d6d7bfa9d --- /dev/null +++ b/changelogs/client_server/newsfragments/1602.feature @@ -0,0 +1 @@ +Add specification for [MSC3077: Multi-stream VoIP](https://github.com/matrix-org/matrix-spec-proposals/pull/3077]. From 3cfade3cdc447dbde0b9e80c6754a73da7c4a978 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 18 Jul 2023 14:35:38 +0100 Subject: [PATCH 3/5] Add examples --- data/event-schemas/examples/m.call.answer.yaml | 10 +++++++++- data/event-schemas/examples/m.call.invite.yaml | 10 +++++++++- data/event-schemas/examples/m.call.negotiate.yaml | 10 +++++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/data/event-schemas/examples/m.call.answer.yaml b/data/event-schemas/examples/m.call.answer.yaml index 78b488783..ba2cc5d25 100644 --- a/data/event-schemas/examples/m.call.answer.yaml +++ b/data/event-schemas/examples/m.call.answer.yaml @@ -8,6 +8,14 @@ "answer": { "type" : "answer", "sdp" : "v=0\r\no=- 6584580628695956864 2 IN IP4 127.0.0.1[...]" - } + }, + "sdp_stream_metadata": { + "271828182845": { + "purpose": "m.screenshare", + }, + "314159265358": { + "purpose": "m.usermedia", + }, + }, } } diff --git a/data/event-schemas/examples/m.call.invite.yaml b/data/event-schemas/examples/m.call.invite.yaml index 45600001e..4ed440f9f 100644 --- a/data/event-schemas/examples/m.call.invite.yaml +++ b/data/event-schemas/examples/m.call.invite.yaml @@ -9,6 +9,14 @@ "offer": { "type" : "offer", "sdp" : "v=0\r\no=- 6584580628695956864 2 IN IP4 127.0.0.1[...]" - } + }, + "sdp_stream_metadata": { + "271828182845": { + "purpose": "m.screenshare", + }, + "314159265358": { + "purpose": "m.usermedia", + }, + }, } } diff --git a/data/event-schemas/examples/m.call.negotiate.yaml b/data/event-schemas/examples/m.call.negotiate.yaml index f4ad8587e..32bb652d3 100644 --- a/data/event-schemas/examples/m.call.negotiate.yaml +++ b/data/event-schemas/examples/m.call.negotiate.yaml @@ -9,6 +9,14 @@ "offer": { "type" : "offer", "sdp" : "v=0\r\no=- 6584580628695956864 2 IN IP4 127.0.0.1[...]" - } + }, + "sdp_stream_metadata": { + "271828182845": { + "purpose": "m.screenshare", + }, + "314159265358": { + "purpose": "m.usermedia", + }, + }, } } From ffac074f3d283fdc4877dcca9ace5c9110272800 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 18 Jul 2023 14:44:57 +0100 Subject: [PATCH 4/5] Maybe make example validator happy? --- data/event-schemas/examples/m.call.answer.yaml | 2 +- data/event-schemas/examples/m.call.invite.yaml | 2 +- data/event-schemas/examples/m.call.negotiate.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data/event-schemas/examples/m.call.answer.yaml b/data/event-schemas/examples/m.call.answer.yaml index ba2cc5d25..b3eb1be42 100644 --- a/data/event-schemas/examples/m.call.answer.yaml +++ b/data/event-schemas/examples/m.call.answer.yaml @@ -16,6 +16,6 @@ "314159265358": { "purpose": "m.usermedia", }, - }, + } } } diff --git a/data/event-schemas/examples/m.call.invite.yaml b/data/event-schemas/examples/m.call.invite.yaml index 4ed440f9f..9958cfe7d 100644 --- a/data/event-schemas/examples/m.call.invite.yaml +++ b/data/event-schemas/examples/m.call.invite.yaml @@ -17,6 +17,6 @@ "314159265358": { "purpose": "m.usermedia", }, - }, + } } } diff --git a/data/event-schemas/examples/m.call.negotiate.yaml b/data/event-schemas/examples/m.call.negotiate.yaml index 32bb652d3..fc79321f2 100644 --- a/data/event-schemas/examples/m.call.negotiate.yaml +++ b/data/event-schemas/examples/m.call.negotiate.yaml @@ -17,6 +17,6 @@ "314159265358": { "purpose": "m.usermedia", }, - }, + } } } From 4e41985f9d806eea274aab40d9a371723de77b28 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 18 Jul 2023 15:20:53 +0100 Subject: [PATCH 5/5] Fix example JSON --- data/event-schemas/examples/m.call.answer.yaml | 6 +++--- data/event-schemas/examples/m.call.invite.yaml | 6 +++--- data/event-schemas/examples/m.call.negotiate.yaml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/data/event-schemas/examples/m.call.answer.yaml b/data/event-schemas/examples/m.call.answer.yaml index b3eb1be42..8a6273603 100644 --- a/data/event-schemas/examples/m.call.answer.yaml +++ b/data/event-schemas/examples/m.call.answer.yaml @@ -11,11 +11,11 @@ }, "sdp_stream_metadata": { "271828182845": { - "purpose": "m.screenshare", + "purpose": "m.screenshare" }, "314159265358": { - "purpose": "m.usermedia", - }, + "purpose": "m.usermedia" + } } } } diff --git a/data/event-schemas/examples/m.call.invite.yaml b/data/event-schemas/examples/m.call.invite.yaml index 9958cfe7d..9547854b5 100644 --- a/data/event-schemas/examples/m.call.invite.yaml +++ b/data/event-schemas/examples/m.call.invite.yaml @@ -12,11 +12,11 @@ }, "sdp_stream_metadata": { "271828182845": { - "purpose": "m.screenshare", + "purpose": "m.screenshare" }, "314159265358": { - "purpose": "m.usermedia", - }, + "purpose": "m.usermedia" + } } } } diff --git a/data/event-schemas/examples/m.call.negotiate.yaml b/data/event-schemas/examples/m.call.negotiate.yaml index fc79321f2..3120ee543 100644 --- a/data/event-schemas/examples/m.call.negotiate.yaml +++ b/data/event-schemas/examples/m.call.negotiate.yaml @@ -12,11 +12,11 @@ }, "sdp_stream_metadata": { "271828182845": { - "purpose": "m.screenshare", + "purpose": "m.screenshare" }, "314159265358": { - "purpose": "m.usermedia", - }, + "purpose": "m.usermedia" + } } } }