Add support for multi-stream VoIP

changelogs/client_server/newsfragments/1735.feature

@@ -0,0 +1 @@
+Add support for multi-stream VoIP, as per [MSC3077](https://github.com/matrix-org/matrix-spec-proposals/pull/3077).

content/client-server-api/modules/voip_events.md

@@ -171,18 +171,34 @@ In response to an incoming invite, a client may do one of several things:
 
 ##### Streams
 
-Clients are expected to send one stream with one track of kind `audio` (creating a
-voice call). They can optionally send a second track in the same stream of kind
-`video` (creating a video call).
-
-Clients implementing this specification use the first stream and will ignore
-any streamless tracks. Note that in the JavaScript WebRTC API, this means
-`addTrack()` must be passed two parameters: a track and a stream, not just a
-track, and in a video call the stream must be the same for both audio and video
-track.
-
-A client may send other streams and tracks but the behaviour of the other party
-with respect to presenting such streams and tracks is undefined.
+Clients may send more than one stream in a VoIP call. Metadata can be included in
+the [`m.call.invite`](/client-server-api/#mcallinvite), [`m.call.answer`](/client-server-api/#mcallanswer)
+and [`m.call.negotiate`](/client-server-api/#mcallnegotiate) events to describe
+the streams being sent, using the `sdp_stream_metadata` property.
+
+`sdp_stream_metadata` maps from the `id` of a stream in the session description, 
+to metadata about that stream.  Currently only one property is defined for the
+metadata. This is `purpose`, which should be a string indicating the purpose of
+the stream. The following `purpose`s are defined:
+
+*  `m.usermedia` - stream that contains the webcam and/or microphone tracks
+*  `m.screenshare` - stream with the screen-sharing tracks
+
+If an incoming stream is not described in `sdp_stream_metadata` and
+`sdp_stream_metadata` is present, the stream should be ignored. If a stream has
+a `purpose` of an unknown type (i.e. not `m.usermedia` or `m.screenshare`), it
+should be ignored.
+
+Clients implementing this specification will ignore any streamless tracks. Note
+that in the JavaScript WebRTC API, this means `addTrack()` must be passed two
+parameters: a track and a stream, not just a track, and in a video call the
+stream must be the same for both audio and video track.
+
+For backwards compatibility, if `sdp_stream_metadata` is not present in the
+initial [`m.call.invite`](/client-server-api/#mcallinvite) or [`m.call.answer`](/client-server-api/#mcallanswer)
+event sent by the other party, the client should ignore any new incoming streams
+(i.e. it should use the first one) and it shouldn't send more than one stream
+(i.e. clients cannot send a video feed and a screenshare at the same time).
 
 ##### Invitees
 The `invitee` field should be added whenever the call is intended for one

data/event-schemas/examples/m.call.answer.yaml

@@ -8,6 +8,14 @@
     "answer": {
       "type" : "answer",
       "sdp" : "v=0\r\no=- 6584580628695956864 2 IN IP4 127.0.0.1[...]"
+    },
+    "sdp_stream_metadata": {
+        "271828182845": {
+            "purpose": "m.screenshare"
+        },
+        "314159265358": {
+            "purpose": "m.usermedia"
+        }
     }
   }
 }

data/event-schemas/examples/m.call.invite.yaml

@@ -9,6 +9,14 @@
     "offer": {
       "type" : "offer",
       "sdp" : "v=0\r\no=- 6584580628695956864 2 IN IP4 127.0.0.1[...]"
+    },
+    "sdp_stream_metadata": {
+        "271828182845": {
+            "purpose": "m.screenshare"
+        },
+        "314159265358": {
+            "purpose": "m.usermedia"
+        }
     }
   }
 }

data/event-schemas/examples/m.call.negotiate.yaml

@@ -9,6 +9,14 @@
     "description": {
       "type" : "offer",
       "sdp" : "v=0\r\no=- 6584580628695956864 2 IN IP4 127.0.0.1[...]"
+    },
+    "sdp_stream_metadata": {
+        "271828182845": {
+            "purpose": "m.screenshare"
+        },
+        "314159265358": {
+            "purpose": "m.usermedia"
+        }
     }
   }
 }

data/event-schemas/schema/core-event-schema/sdp_stream_metadata.yaml

@@ -0,0 +1,27 @@
+type: object
+x-addedInMatrixVersion: "1.10"
+description: |-
+  Metadata describing the [streams](/client-server-api/#streams) that will be
+  sent.
+
+  This is a map of stream ID to metadata about the stream.
+additionalProperties:
+  type: object
+  title: StreamMetadata
+  description: Metadata describing a stream.
+  properties:
+    purpose:
+      type: string
+      enum:
+        - m.usermedia
+        - m.screenshare
+      description: |-
+        The purpose of the stream.
+
+        The possible values are:
+
+        * `m.usermedia` - stream that contains the webcam and/or microphone
+          tracks
+        * `m.screenshare` - stream with the screen-sharing tracks      
+  required:
+    - purpose

data/event-schemas/schema/m.call.answer.yaml

@@ -27,6 +27,9 @@
                         }
                     },
                     "required": ["type", "sdp"]
+                },
+                "sdp_stream_metadata": {
+                    "$ref": "core-event-schema/sdp_stream_metadata.yaml"
                 }
             },
             "required": ["answer"]

data/event-schemas/schema/m.call.invite.yaml

@@ -35,7 +35,10 @@
                 "invitee": {
                     "type": "string",
                     "description": "The ID of the user being called. If omitted, any user in the room can answer.",
-                    "x-addedInMatrixVersion": "1.7",
+                    "x-addedInMatrixVersion": "1.7"
+                },
+                "sdp_stream_metadata": {
+                    "$ref": "core-event-schema/sdp_stream_metadata.yaml"
                 }
             },
             "required": ["offer", "lifetime"]

data/event-schemas/schema/m.call.negotiate.yaml

@@ -63,6 +63,8 @@ properties:
         type: integer
         description: The time in milliseconds that the negotiation is valid for.
           Once the negotiation age exceeds this value, clients should discard it.
+      sdp_stream_metadata:
+        $ref: core-event-schema/sdp_stream_metadata.yaml
     required:
     - description
     - lifetime