From 45cb682f827cd367aa4187ab63251d765624e87b Mon Sep 17 00:00:00 2001 From: Renan Dincer Date: Thu, 27 Feb 2025 11:02:11 -0500 Subject: [PATCH 1/7] Calls: Introduce simulcast API --- .../realtime/static/calls-api-2024-05-21.yaml | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/public/realtime/static/calls-api-2024-05-21.yaml b/public/realtime/static/calls-api-2024-05-21.yaml index 678bd9f2fc90e28..6c1297eb79d1fe6 100644 --- a/public/realtime/static/calls-api-2024-05-21.yaml +++ b/public/realtime/static/calls-api-2024-05-21.yaml @@ -140,6 +140,33 @@ paths: sessionId: 2a45361d5fd7cc14eface0587c276c94 trackName: generated-audio kind: "audio" + simulcast_track: + description: Share a track with simulcast configuration + value: + sessionDescription: + sdp: | + v=0 + o=- 0 0 IN IP4 127.0.0.1 + s=- + c=IN IP4 127.0.0.1 + t=0 0 + m=audio 4000 RTP/AVP 111 + a=rtpmap:111 OPUS/48000/2 + m=video 4002 RTP/AVP 96 + a=rtpmap:96 VP8/90000 + a=simulcast:send f;h;q + a=rid:f send + a=rid:h send + a=rid:q send + ... + type: offer + tracks: + - location: local + trackName: simulcast-video-track + mid: "1" + simulcast: + preferredRid: "h" + preferredRidNotAvailable: "leastbandwidth" security: - secret: [] parameters: @@ -329,6 +356,75 @@ paths: requiresImmediateRenegotiation: false tracks: - mid: "7" + /apps/{appId}/sessions/{sessionId}/tracks/change: + post: + tags: + - Change tracks + summary: Change tracks by reusing existing transceivers + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ChangeTracksRequest" + examples: + reuse_transceiver: + description: Reuse an existing transceiver for a new track + value: + tracks: + some-track-name: + location: "remote" + sessionId: "2a45361d5fd7cc14eface0587c276c94" + trackName: "other-track-name" + mid: "7" + reuse_with_simulcast: + description: Reuse an existing transceiver with simulcast preferences + value: + tracks: + trackID2: + location: "remote" + sessionId: "2a45361d5fd7cc14eface0587c276c94" + trackName: "simulcast-track" + mid: "8" + simulcast: + preferredRid: "h" + preferredRidNotAvailable: "leastbandwidth" + security: + - secret: [] + parameters: + - in: path + name: appId + schema: + type: string + required: true + description: WebRTC application ID + - in: path + name: sessionId + schema: + type: string + required: true + description: Current PeerConnection session ID + responses: + "200": + description: OK + headers: + vary: + schema: + type: string + example: Origin + content: + application/json: + schema: + $ref: "#/components/schemas/ChangeTracksResponse" + examples: + success: + value: + requiresImmediateRenegotiation: false + tracks: + trackID1: + mid: "7" + sessionId: "2a45361d5fd7cc14eface0587c276c94" + trackName: "new-track-name" + /apps/{appId}/sessions/{sessionId}: get: tags: @@ -412,6 +508,21 @@ components: kind: type: string description: Give a hint to the SFU about the transceiver kind. This is required when the SFU generates the offer + simulcast: + type: object + description: Simulcast configuration for the track + properties: + preferredRid: + type: string + description: Preferred RID (Resolution ID) for simulcast streams + preferredRidNotAvailable: + type: string + enum: + - leastbandwidth + - none + - error + default: leastbandwidth + description: Fallback strategy when preferred RID is not available anymore from the remote peer CloseTrackObject: type: object properties: @@ -543,3 +654,34 @@ components: type: string sessionDescription: $ref: "#/components/schemas/SessionDescription" + ChangeTracksRequest: + type: object + properties: + tracks: + type: object + additionalProperties: + allOf: + - $ref: "#/components/schemas/TrackObject" + description: Map of track IDs to track objects for changing tracks + sessionDescription: + $ref: "#/components/schemas/SessionDescription" + ChangeTracksResponse: + type: object + properties: + errorCode: + type: string + errorDescription: + type: string + requiresImmediateRenegotiation: + type: boolean + tracks: + type: object + additionalProperties: + allOf: + - $ref: "#/components/schemas/TrackObject" + - properties: + errorCode: + type: string + errorDescription: + type: string + description: Map of track IDs to track objects with results From 6fcc5284053714d14ebfd364e142fb790ccba23f Mon Sep 17 00:00:00 2001 From: Renan Dincer Date: Fri, 28 Feb 2025 14:32:29 -0500 Subject: [PATCH 2/7] Update Calls simulcast API definition after feedback on 2025-02-27 --- .../realtime/static/calls-api-2024-05-21.yaml | 72 ++++++++++--------- 1 file changed, 40 insertions(+), 32 deletions(-) diff --git a/public/realtime/static/calls-api-2024-05-21.yaml b/public/realtime/static/calls-api-2024-05-21.yaml index 6c1297eb79d1fe6..91cc27ab11733ca 100644 --- a/public/realtime/static/calls-api-2024-05-21.yaml +++ b/public/realtime/static/calls-api-2024-05-21.yaml @@ -140,33 +140,16 @@ paths: sessionId: 2a45361d5fd7cc14eface0587c276c94 trackName: generated-audio kind: "audio" - simulcast_track: - description: Share a track with simulcast configuration + remote_track_with_simulcast: + description: Pull a remote track with simulcast preferences value: - sessionDescription: - sdp: | - v=0 - o=- 0 0 IN IP4 127.0.0.1 - s=- - c=IN IP4 127.0.0.1 - t=0 0 - m=audio 4000 RTP/AVP 111 - a=rtpmap:111 OPUS/48000/2 - m=video 4002 RTP/AVP 96 - a=rtpmap:96 VP8/90000 - a=simulcast:send f;h;q - a=rid:f send - a=rid:h send - a=rid:q send - ... - type: offer tracks: - - location: local + - location: remote + sessionId: 2a45361d5fd7cc14eface0587c276c94 trackName: simulcast-video-track - mid: "1" simulcast: preferredRid: "h" - preferredRidNotAvailable: "leastbandwidth" + fallbackStrategy: "leastBandwidth" security: - secret: [] parameters: @@ -234,6 +217,31 @@ paths: a=rtpmap:96 VP8/90000 ... type: offer + remote_tracks_with_simulcast: + value: + requiresImmediateRenegotiation: true + tracks: + - sessionId: 2a45361d5fd7cc14eface0587c276c94 + trackName: simulcast-video-track + mid: "5" + simulcast: + preferredRid: "h" + fallbackStrategy: "leastBandwidth" + sessionDescription: + sdp: | + v=0 + o=- 0 0 IN IP4 127.0.0.1 + s=- + c=IN IP4 127.0.0.1 + t=0 0 + m=video 4002 RTP/AVP 96 + a=rtpmap:96 VP8/90000 + a=simulcast:recv f;h;q + a=rid:f recv + a=rid:h recv + a=rid:q recv + ... + type: offer /apps/{appId}/sessions/{sessionId}/renegotiate: put: tags: @@ -356,8 +364,8 @@ paths: requiresImmediateRenegotiation: false tracks: - mid: "7" - /apps/{appId}/sessions/{sessionId}/tracks/change: - post: + /apps/{appId}/sessions/{sessionId}/tracks/update: + put: tags: - Change tracks summary: Change tracks by reusing existing transceivers @@ -387,7 +395,7 @@ paths: mid: "8" simulcast: preferredRid: "h" - preferredRidNotAvailable: "leastbandwidth" + fallbackStrategy: "leastBandwidth" security: - secret: [] parameters: @@ -414,7 +422,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/ChangeTracksResponse" + $ref: "#/components/schemas/UpdateTracksResponse" examples: success: value: @@ -515,14 +523,14 @@ components: preferredRid: type: string description: Preferred RID (Resolution ID) for simulcast streams - preferredRidNotAvailable: + fallbackStrategy: type: string enum: - - leastbandwidth + - auto + - leastBandwidth - none - - error - default: leastbandwidth - description: Fallback strategy when preferred RID is not available anymore from the remote peer + default: auto + description: General fallback strategy when any constraint (like preferredRid, minWidth, maxBandwidth, etc.) cannot be met. 'auto' will select the best quality possible within constraints. CloseTrackObject: type: object properties: @@ -665,7 +673,7 @@ components: description: Map of track IDs to track objects for changing tracks sessionDescription: $ref: "#/components/schemas/SessionDescription" - ChangeTracksResponse: + UpdateTracksResponse: type: object properties: errorCode: From 197cf55b022dacc1eb254d9029cfe2570edd29d6 Mon Sep 17 00:00:00 2001 From: Renan Dincer Date: Fri, 28 Feb 2025 15:53:16 -0500 Subject: [PATCH 3/7] Calls: Make UpdateTracksRequest schema to be a array of TrackObject --- .../realtime/static/calls-api-2024-05-21.yaml | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/public/realtime/static/calls-api-2024-05-21.yaml b/public/realtime/static/calls-api-2024-05-21.yaml index 91cc27ab11733ca..16002e766c7772d 100644 --- a/public/realtime/static/calls-api-2024-05-21.yaml +++ b/public/realtime/static/calls-api-2024-05-21.yaml @@ -373,14 +373,13 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/ChangeTracksRequest" + $ref: "#/components/schemas/UpdateTracksRequest" examples: reuse_transceiver: description: Reuse an existing transceiver for a new track value: tracks: - some-track-name: - location: "remote" + - location: "remote" sessionId: "2a45361d5fd7cc14eface0587c276c94" trackName: "other-track-name" mid: "7" @@ -388,8 +387,7 @@ paths: description: Reuse an existing transceiver with simulcast preferences value: tracks: - trackID2: - location: "remote" + - location: "remote" sessionId: "2a45361d5fd7cc14eface0587c276c94" trackName: "simulcast-track" mid: "8" @@ -428,8 +426,7 @@ paths: value: requiresImmediateRenegotiation: false tracks: - trackID1: - mid: "7" + - mid: "7" sessionId: "2a45361d5fd7cc14eface0587c276c94" trackName: "new-track-name" @@ -673,6 +670,16 @@ components: description: Map of track IDs to track objects for changing tracks sessionDescription: $ref: "#/components/schemas/SessionDescription" + UpdateTracksRequest: + type: object + properties: + tracks: + type: array + items: + $ref: "#/components/schemas/TrackObject" + description: Array of track objects for updating tracks + sessionDescription: + $ref: "#/components/schemas/SessionDescription" UpdateTracksResponse: type: object properties: @@ -683,8 +690,8 @@ components: requiresImmediateRenegotiation: type: boolean tracks: - type: object - additionalProperties: + type: array + items: allOf: - $ref: "#/components/schemas/TrackObject" - properties: @@ -692,4 +699,4 @@ components: type: string errorDescription: type: string - description: Map of track IDs to track objects with results + description: Array of track objects with results From bd523eccb5f8b75c67e82cf50dbe2b74f565a996 Mon Sep 17 00:00:00 2001 From: Renan Dincer Date: Tue, 4 Mar 2025 14:14:46 -0500 Subject: [PATCH 4/7] Calls: Replace Simulcast API fallbackStrategy with priorityOrdering and ridUnavailableStrategy --- .../realtime/static/calls-api-2024-05-21.yaml | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/public/realtime/static/calls-api-2024-05-21.yaml b/public/realtime/static/calls-api-2024-05-21.yaml index 16002e766c7772d..bf972f455e101f3 100644 --- a/public/realtime/static/calls-api-2024-05-21.yaml +++ b/public/realtime/static/calls-api-2024-05-21.yaml @@ -149,7 +149,8 @@ paths: trackName: simulcast-video-track simulcast: preferredRid: "h" - fallbackStrategy: "leastBandwidth" + priorityOrdering: "asciibetical" + ridUnavailableStrategy: "nextPriority" security: - secret: [] parameters: @@ -226,7 +227,8 @@ paths: mid: "5" simulcast: preferredRid: "h" - fallbackStrategy: "leastBandwidth" + priorityOrdering: "asciibetical" + ridUnavailableStrategy: "nextPriority" sessionDescription: sdp: | v=0 @@ -393,7 +395,8 @@ paths: mid: "8" simulcast: preferredRid: "h" - fallbackStrategy: "leastBandwidth" + priorityOrdering: "asciibetical" + ridUnavailableStrategy: "nextPriority" security: - secret: [] parameters: @@ -520,14 +523,20 @@ components: preferredRid: type: string description: Preferred RID (Resolution ID) for simulcast streams - fallbackStrategy: + priorityOrdering: type: string enum: - - auto - - leastBandwidth - none - default: auto - description: General fallback strategy when any constraint (like preferredRid, minWidth, maxBandwidth, etc.) cannot be met. 'auto' will select the best quality possible within constraints. + - asciibetical + default: none + description: Controls what happens if there is not enough network resources available to send the preferredRid. 'none' means keep sending even if not enough bandwidth, 'asciibetical' uses a-z order to determine priority where a is most desirable and z is least desirable. + ridUnavailableStrategy: + type: string + enum: + - none + - nextPriority + default: none + description: Controls what happens when the rid currently being used or preferredRid is no longer being sent by the publisher. 'none' means do nothing, 'nextPriority' uses the next on the priorityOrdering. CloseTrackObject: type: object properties: From 6604e30ad770b94c7a21bb1299fc42265abc6903 Mon Sep 17 00:00:00 2001 From: Renan Dincer Date: Tue, 4 Mar 2025 14:51:10 -0500 Subject: [PATCH 5/7] Calls: Update the Calls Simulcast API after Nils' comments --- public/realtime/static/calls-api-2024-05-21.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/realtime/static/calls-api-2024-05-21.yaml b/public/realtime/static/calls-api-2024-05-21.yaml index bf972f455e101f3..99c871e71a305a5 100644 --- a/public/realtime/static/calls-api-2024-05-21.yaml +++ b/public/realtime/static/calls-api-2024-05-21.yaml @@ -386,7 +386,7 @@ paths: trackName: "other-track-name" mid: "7" reuse_with_simulcast: - description: Reuse an existing transceiver with simulcast preferences + description: Reuse an existing transceiver with new simulcast preferences value: tracks: - location: "remote" @@ -522,7 +522,7 @@ components: properties: preferredRid: type: string - description: Preferred RID (Resolution ID) for simulcast streams + description: Preferred RID for simulcast streams priorityOrdering: type: string enum: From 3cf389778b560f9dbeb50a49817a61171085f00a Mon Sep 17 00:00:00 2001 From: Renan Dincer Date: Tue, 4 Mar 2025 16:41:55 -0500 Subject: [PATCH 6/7] Calls: Add Simulcast documentation text --- src/content/docs/realtime/simulcast.mdx | 76 +++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/content/docs/realtime/simulcast.mdx diff --git a/src/content/docs/realtime/simulcast.mdx b/src/content/docs/realtime/simulcast.mdx new file mode 100644 index 000000000000000..a1b970c994e6ab6 --- /dev/null +++ b/src/content/docs/realtime/simulcast.mdx @@ -0,0 +1,76 @@ +--- +pcx_content_type: get-started +title: Simulcast +sidebar: + order: 8 +--- + +Simulcast is a feature of WebRTC that allows a publisher to send multiple streams of the same media at different qualities. For example, this is useful for scenarios where you want to send a high quality stream for desktop users and a lower quality stream for mobile users. + +```mermaid +graph LR + A[Publisher] -->|Low quality| B[Cloudflare Calls SFU] + A -->|Medium quality| B + A -->|High quality| B +B -->|Low quality| C@{ shape: procs, label: "Subscribers"} +B -->|Medium quality| D@{ shape: procs, label: "Subscribers"} +B -->|High quality| E@{ shape: procs, label: "Subscribers"} + +``` + +### How it works + +Simulcast in WebRTC allows a single media source, like a camera or screen share, to be encoded at multiple quality levels and sent simultaneously, which is beneficial for subscribers with varying network conditions and device capabilities. The media source is encoded into multiple streams, each identified by RIDs (RTP Stream Identifiers) for different quality levels, such as low, medium, and high. These simulcast streams are described in the SDP you send to Cloudflare Calls SFU. It's the responsibility of the Cloudflare Calls SFU to ensure that the appropriate quality stream is delivered to each subscriber based on their network conditions and device capabilities. + +Cloudflare Calls SFU will automatically handle the simulcast configuration based on the SDP you send to it from the publisher. The SFU will then automatically switch between the different quality levels based on the subscriber's network conditions. You can control the quality switching behavior using the `simulcast` configuration object when you send an API call to start pulling a remote track. + +### Quality Control + +The `simulcast` configuration object in the API call when you start pulling a remote track allows you to specify: + +- `preferredRid`: The preferred stream (RID for the simulcast stream. [RIDs can be specified by the publisher.](https://developer.mozilla.org/en-US/docs/Web/API/RTCRtpSender/setParameters#encodings)) +- `priorityOrdering`: Controls how the SFU handles bandwidth constraints + - `none`: Keep sending the preferred layer even if there's not enough bandwidth + - `asciibetical`: Use alphabetical ordering (a-z) to determine priority, where 'a' is most desirable and 'z' is least desirable +- `ridUnavailableStrategy`: Controls what happens when the preferred RID is no longer available, for example when the publisher stops sending it + + - `none`: Do nothing + - `nextPriority`: Switch to the next available RID based on the priority ordering + + You will likely want to order the asciibetical RIDs based on your desired metric, such as higest resoltion to lowest or highest bandwidth to lowest. + +### Bandwidth Management across media tracks + +Cloudflare Calls treats all media tracks equally at the transport level. For example, if you have multiple video tracks (cameras, screen shares, etc.), they all have equal priority for bandwidth allocation. This means: + +1. Each track's simulcast configuration is handled independently +1. The SFU performs automatic bandwidth estimation and layer switching based on network conditions independently for each track + +### Layer Switching Behavior + +When a layer switch is requested (through updating `preferredRid`) with the `/tracks/update` API: + +1. The SFU will automatically generate a Picture Loss Indication (PLI) +2. Layer switching only occurs when a keyframe arrives on the target layer +3. PLI generation is debounced to prevent excessive requests + +### Publisher Configuration + +For publishers (local tracks), you only need to include the simulcast attributes in your SDP. The SFU will automatically handle the simulcast configuration based on the SDP. For example, the SDP should contain a section like this: + +```sdp +a=simulcast:send f;h;q +a=rid:f send +a=rid:h send +a=rid:q send +``` + +## Example + +Here's an example of how to use simulcast with Cloudflare Calls: + +1. Create a new local track with simulcast configuration. There should be a section in the SDP with `a=simulcast:send`. +2. Use the [Cloudflare Calls API](/calls/https-api) to push this local track, by calling the /tracks/new endpoint. +3. Use the [Cloudflare Calls API](/calls/https-api) to start pulling a remote track (from another browser or device), by calling the /tracks/new endpoint and specifying the `simulcast` configuration object along with the remote track ID you get from step 2. + +For more examples, check out the [Calls Examples GitHub repository](https://github.com/cloudflare/calls-examples/tree/main/simulcast). From 2b5ea9a90f35b118669e2d66b05c77a25c0e2c28 Mon Sep 17 00:00:00 2001 From: Renan Dincer Date: Tue, 8 Apr 2025 19:13:28 -0400 Subject: [PATCH 7/7] Update src/content/docs/calls/simulcast.mdx Co-authored-by: Kevin Kipp --- src/content/docs/realtime/simulcast.mdx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/content/docs/realtime/simulcast.mdx b/src/content/docs/realtime/simulcast.mdx index a1b970c994e6ab6..88514f612ffe7fb 100644 --- a/src/content/docs/realtime/simulcast.mdx +++ b/src/content/docs/realtime/simulcast.mdx @@ -65,6 +65,17 @@ a=rid:h send a=rid:q send ``` +You can include these by specifying `sendEncodings` when creating the transceiver: + +```js +const transceiver = peerConnection.addTransceiver(track, { + direction: "sendonly", + sendEncodings: [ + { scaleResolutionDownBy: 1, rid: "a" }, + { scaleResolutionDownBy: 2, rid: "b" }, + { scaleResolutionDownBy: 4, rid: "c" } + ] +}); ## Example Here's an example of how to use simulcast with Cloudflare Calls: