diff --git a/changelogs/client_server/newsfragments/1602.feature b/changelogs/client_server/newsfragments/1602.feature new file mode 100644 index 000000000..d6d7bfa9d --- /dev/null +++ b/changelogs/client_server/newsfragments/1602.feature @@ -0,0 +1 @@ +Add specification for [MSC3077: Multi-stream VoIP](https://github.com/matrix-org/matrix-spec-proposals/pull/3077]. diff --git a/content/client-server-api/modules/voip_events.md b/content/client-server-api/modules/voip_events.md index 102e3dcd7..415b8a05c 100644 --- a/content/client-server-api/modules/voip_events.md +++ b/content/client-server-api/modules/voip_events.md @@ -171,18 +171,34 @@ In response to an incoming invite, a client may do one of several things: ##### Streams -Clients are expected to send one stream with one track of kind `audio` (creating a -voice call). They can optionally send a second track in the same stream of kind -`video` (creating a video call). - -Clients implementing this specification use the first stream and will ignore -any streamless tracks. Note that in the JavaScript WebRTC API, this means -`addTrack()` must be passed two parameters: a track and a stream, not just a -track, and in a video call the stream must be the same for both audio and video -track. - -A client may send other streams and tracks but the behaviour of the other party -with respect to presenting such streams and tracks is undefined. +Clients may send more than one stream in a VoIP call. Metadata can be included in +the `m.call.invite`, `m.call.answer` and `m.call.negotiate` events to describe the +streams being sent. This is the `sdp_stream_metadata` field. + +This field is an object in which each key is one stream `id` in the session +description. The values are objects with the following fields: + + * `purpose` - a string indicating the purpose of the stream. For compatibility + between client the following values are defined: + * `m.usermedia` - stream that contains the webcam and/or microphone tracks + * `m.screenshare` - stream with the screen-sharing tracks + +If an incoming stream is not described in `sdp_stream_metadata` and +`sdp_stream_metadata` is present, the stream should be ignored. If a stream has +a `purpose` of an unknown type (i.e. not `m.usermedia` or `m.screenshare`), it +should be ignored. + +Clients implementing this specification will ignore any streamless tracks. Note +that in the JavaScript WebRTC API, this means `addTrack()` must be passed two +parameters: a track and a stream, not just a track, and in a video call the +stream must be the same for both audio and video track. + +During the initial invite and answer exchange clients find out if the field +`sdp_stream_metadata` is missing. For backwards compatibility if it is not +present in the event sent by the opponent, the client should ignore any new +incoming streams (i.e. it should use the first one) and it shouldn't send more +than one stream (i.e. clients cannot send a video feed and a screenshare at the +same time, as is the case in current clients). ##### Invitees The `invitee` field should be added whenever the call is intended for one diff --git a/data/event-schemas/examples/m.call.answer.yaml b/data/event-schemas/examples/m.call.answer.yaml index 78b488783..8a6273603 100644 --- a/data/event-schemas/examples/m.call.answer.yaml +++ b/data/event-schemas/examples/m.call.answer.yaml @@ -8,6 +8,14 @@ "answer": { "type" : "answer", "sdp" : "v=0\r\no=- 6584580628695956864 2 IN IP4 127.0.0.1[...]" + }, + "sdp_stream_metadata": { + "271828182845": { + "purpose": "m.screenshare" + }, + "314159265358": { + "purpose": "m.usermedia" + } } } } diff --git a/data/event-schemas/examples/m.call.invite.yaml b/data/event-schemas/examples/m.call.invite.yaml index 45600001e..9547854b5 100644 --- a/data/event-schemas/examples/m.call.invite.yaml +++ b/data/event-schemas/examples/m.call.invite.yaml @@ -9,6 +9,14 @@ "offer": { "type" : "offer", "sdp" : "v=0\r\no=- 6584580628695956864 2 IN IP4 127.0.0.1[...]" + }, + "sdp_stream_metadata": { + "271828182845": { + "purpose": "m.screenshare" + }, + "314159265358": { + "purpose": "m.usermedia" + } } } } diff --git a/data/event-schemas/examples/m.call.negotiate.yaml b/data/event-schemas/examples/m.call.negotiate.yaml index f4ad8587e..3120ee543 100644 --- a/data/event-schemas/examples/m.call.negotiate.yaml +++ b/data/event-schemas/examples/m.call.negotiate.yaml @@ -9,6 +9,14 @@ "offer": { "type" : "offer", "sdp" : "v=0\r\no=- 6584580628695956864 2 IN IP4 127.0.0.1[...]" + }, + "sdp_stream_metadata": { + "271828182845": { + "purpose": "m.screenshare" + }, + "314159265358": { + "purpose": "m.usermedia" + } } } } diff --git a/data/event-schemas/schema/m.call.answer.yaml b/data/event-schemas/schema/m.call.answer.yaml index 163690be2..44e40dd21 100644 --- a/data/event-schemas/schema/m.call.answer.yaml +++ b/data/event-schemas/schema/m.call.answer.yaml @@ -27,6 +27,23 @@ } }, "required": ["type", "sdp"] + }, + "sdp_stream_metadata": { + "type": "object", + "title": "SDP Stream Metadata", + "x-addedInMatrixVersion": "1.7", + "description": "Object describing the streams that will be sent", + "additionalProperties": { + "type": "object", + "title": "Stream Metadata Block", + "properties": { + "purpose": { + "type": "string", + "enum": ["m.usermedia", "m.screenshare"] + } + }, + "required": ["purpose"] + } } }, "required": ["answer"] diff --git a/data/event-schemas/schema/m.call.invite.yaml b/data/event-schemas/schema/m.call.invite.yaml index 72020b266..8bdde9b85 100644 --- a/data/event-schemas/schema/m.call.invite.yaml +++ b/data/event-schemas/schema/m.call.invite.yaml @@ -35,7 +35,24 @@ "invitee": { "type": "string", "description": "The ID of the user being called. If omitted, any user in the room can answer.", + "x-addedInMatrixVersion": "1.7" + }, + "sdp_stream_metadata": { + "type": "object", + "title": "SDP Stream Metadata", "x-addedInMatrixVersion": "1.7", + "description": "Object describing the streams that will be sent", + "additionalProperties": { + "type": "object", + "title": "Stream Metadata Block", + "properties": { + "purpose": { + "type": "string", + "enum": ["m.usermedia", "m.screenshare"] + } + }, + "required": ["purpose"] + } } }, "required": ["offer", "lifetime"] diff --git a/data/event-schemas/schema/m.call.negotiate.yaml b/data/event-schemas/schema/m.call.negotiate.yaml index abc5ef1d8..f34d6502e 100644 --- a/data/event-schemas/schema/m.call.negotiate.yaml +++ b/data/event-schemas/schema/m.call.negotiate.yaml @@ -64,6 +64,23 @@ properties: Once the invite age exceeds this value, clients should discard it. They should also no longer show the call as awaiting an answer in the UI. + sdp_stream_metadata: + type: object + title: SDP Stream Metadata + x-addedInMatrixVersion: '1.7' + description: Object describing the streams that will be sent + additionalProperties: + type: object + title: Stream Metadata Block + properties: + purpose: + type: string + enum: + - m.usermedia + - m.screenshare + required: + - purpose + required: - offer - lifetime