Multi-codec v2 (livekit#275)

Support for `VP9`, `AV1`, `VP8`, `H264` codecs.
v2 compatible version of
livekit#176

Sources/LiveKit/Core/Engine+SignalClientDelegate.swift

@@ -96,7 +96,7 @@ extension Engine: SignalClientDelegate {
     func signalClient(_: SignalClient, didUpdateConnectionQuality _: [Livekit_ConnectionQualityInfo]) {}
     func signalClient(_: SignalClient, didUpdateRemoteMute _: String, muted _: Bool) {}
     func signalClient(_: SignalClient, didUpdateTrackStreamStates _: [Livekit_StreamStateInfo]) {}
-    func signalClient(_: SignalClient, didUpdateTrack _: String, subscribedQualities _: [Livekit_SubscribedQuality]) {}
+    func signalClient(_: SignalClient, didUpdateTrack _: String, subscribedQualities _: [Livekit_SubscribedQuality], subscribedCodecs _: [Livekit_SubscribedCodec]) {}
     func signalClient(_: SignalClient, didUpdateSubscriptionPermission _: Livekit_SubscriptionPermissionUpdate) {}
     func signalClient(_: SignalClient, didReceiveLeave _: Bool, reason _: Livekit_DisconnectReason) {}
 }

Sources/LiveKit/Core/Engine+WebRTC.swift

@@ -75,6 +75,9 @@ extension Engine {
 
     static let audioProcessingModule: LKRTCDefaultAudioProcessingModule = .init()
 
+    static let videoSenderCapabilities = peerConnectionFactory.rtpSenderCapabilities(for: .video)
+    static let audioSenderCapabilities = peerConnectionFactory.rtpSenderCapabilities(for: .audio)
+
     static let peerConnectionFactory: LKRTCPeerConnectionFactory = {
         logger.log("Initializing SSL...", type: Engine.self)
 
@@ -153,7 +156,8 @@ extension Engine {
     static func createRtpEncodingParameters(rid: String? = nil,
                                             encoding: MediaEncoding? = nil,
                                             scaleDownBy: Double? = nil,
-                                            active: Bool = true) -> LKRTCRtpEncodingParameters
+                                            active: Bool = true,
+                                            scalabilityMode: ScalabilityMode? = nil) -> LKRTCRtpEncodingParameters
     {
         let result = DispatchQueue.liveKitWebRTC.sync { LKRTCRtpEncodingParameters() }
 
@@ -173,6 +177,10 @@ extension Engine {
             }
         }
 
+        if let scalabilityMode {
+            result.scalabilityMode = scalabilityMode.rawStringValue
+        }
+
         return result
     }
 }

Sources/LiveKit/Core/Room+SignalClientDelegate.swift

@@ -33,10 +33,35 @@ extension Room: SignalClientDelegate {
         }
     }
 
-    func signalClient(_: SignalClient, didUpdateTrack trackSid: String, subscribedQualities: [Livekit_SubscribedQuality]) {
-        log("qualities: \(subscribedQualities.map { String(describing: $0) }.joined(separator: ", "))")
+    func signalClient(_: SignalClient, didUpdateTrack trackSid: String, subscribedQualities qualities: [Livekit_SubscribedQuality], subscribedCodecs codecs: [Livekit_SubscribedCodec]) {
+        log("[Publish/Backup] Qualities: \(qualities.map { String(describing: $0) }.joined(separator: ", ")), Codecs: \(codecs.map { String(describing: $0) }.joined(separator: ", "))")
 
-        localParticipant.onSubscribedQualitiesUpdate(trackSid: trackSid, subscribedQualities: subscribedQualities)
+        guard let publication = localParticipant.getTrackPublication(sid: trackSid) else {
+            log("Received subscribed quality update for an unknown track", .warning)
+            return
+        }
+
+        Task {
+            if !codecs.isEmpty {
+                guard let videoTrack = publication.track as? LocalVideoTrack else { return }
+                let missingSubscribedCodecs = try videoTrack._set(subscribedCodecs: codecs)
+
+                if !missingSubscribedCodecs.isEmpty {
+                    log("Missing codecs: \(missingSubscribedCodecs)")
+                    for missingSubscribedCodec in missingSubscribedCodecs {
+                        do {
+                            log("Publishing additional codec: \(missingSubscribedCodec)")
+                            try await localParticipant.publish(additionalVideoCodec: missingSubscribedCodec, for: publication)
+                        } catch {
+                            log("Failed publishing additional codec: \(missingSubscribedCodec), error: \(error)", .error)
+                        }
+                    }
+                }
+
+            } else {
+                localParticipant._set(subscribedQualities: qualities, forTrackSid: trackSid)
+            }
+        }
     }
 
     func signalClient(_: SignalClient, didReceiveJoinResponse joinResponse: Livekit_JoinResponse) {

Sources/LiveKit/Core/SignalClient.swift

@@ -286,11 +286,11 @@ private extension SignalClient {
             notify { $0.signalClient(self, didUpdateTrackStreamStates: states.streamStates) }
 
         case let .subscribedQualityUpdate(update):
-            // ignore 0.15.1
-            if latestJoinResponse?.serverVersion == "0.15.1" {
-                return
-            }
-            notify { $0.signalClient(self, didUpdateTrack: update.trackSid, subscribedQualities: update.subscribedQualities) }
+            notify { $0.signalClient(self,
+                                     didUpdateTrack: update.trackSid,
+                                     subscribedQualities: update.subscribedQualities,
+                                     subscribedCodecs: update.subscribedCodecs) }
+
         case let .subscriptionPermissionUpdate(permissionUpdate):
             notify { $0.signalClient(self, didUpdateSubscriptionPermission: permissionUpdate) }
         case let .refreshToken(token):

Sources/LiveKit/Extensions/CustomStringConvertible.swift

@@ -63,6 +63,12 @@ extension Livekit_SubscribedQuality: CustomStringConvertible {
     }
 }
 
+extension Livekit_SubscribedCodec: CustomStringConvertible {
+    public var description: String {
+        "SubscribedCodec(codec: \(codec), qualities: \(qualities.map { String(describing: $0) }.joined(separator: ", "))"
+    }
+}
+
 extension Livekit_ServerInfo: CustomStringConvertible {
     public var description: String {
         "ServerInfo(edition: \(edition), " +

Sources/LiveKit/Extensions/LKRTCRtpSender.swift

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+
+@_implementationOnly import WebRTC
+
+extension LKRTCRtpSender: Loggable {
+    // ...
+    func _set(subscribedQualities qualities: [Livekit_SubscribedQuality]) {
+        let _parameters = parameters
+        let encodings = _parameters.encodings
+
+        var didUpdate = false
+
+        // For SVC mode...
+        if let firstEncoding = encodings.first,
+           let _ = ScalabilityMode.fromString(firstEncoding.scalabilityMode)
+        {
+            let _enabled = qualities.highest != .off
+            if firstEncoding.isActive != _enabled {
+                firstEncoding.isActive = _enabled
+                didUpdate = true
+            }
+        } else {
+            // For Simulcast...
+            for e in qualities {
+                guard let rid = e.quality.asRID else { continue }
+                guard let encodingforRID = encodings.first(where: { $0.rid == rid }) else { continue }
+
+                if encodingforRID.isActive != e.enabled {
+                    didUpdate = true
+                    encodingforRID.isActive = e.enabled
+                    log("Setting layer \(e.quality) to \(e.enabled)", .info)
+                }
+            }
+
+            // Non simulcast streams don't have RIDs, handle here.
+            if encodings.count == 1, qualities.count >= 1 {
+                let firstEncoding = encodings.first!
+                let firstQuality = qualities.first!
+
+                if firstEncoding.isActive != firstQuality.enabled {
+                    didUpdate = true
+                    firstEncoding.isActive = firstQuality.enabled
+                    log("Setting layer \(firstQuality.quality) to \(firstQuality.enabled)", .info)
+                }
+            }
+        }
+
+        if didUpdate {
+            parameters = _parameters
+        }
+    }
+}

Sources/LiveKit/Extensions/RTCRtpTransceiver.swift

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2023 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Foundation
+
+@_implementationOnly import WebRTC
+
+extension LKRTCRtpTransceiver: Loggable {
+    /// Attempts to set preferred video codec.
+    func set(preferredVideoCodec codec: VideoCodec, exceptCodec: VideoCodec? = nil) {
+        // Get list of supported codecs...
+        let allVideoCodecs = Engine.videoSenderCapabilities.codecs
+
+        // Get the RTCRtpCodecCapability of the preferred codec
+        let preferredCodecCapability = allVideoCodecs.first { $0.name.lowercased() == codec.id }
+
+        // Get list of capabilities other than the preferred one
+        let otherCapabilities = allVideoCodecs.filter {
+            $0.name.lowercased() != codec.id && $0.name.lowercased() != exceptCodec?.id
+        }
+
+        // Bring preferredCodecCapability to the front and combine all capabilities
+        let combinedCapabilities = [preferredCodecCapability] + otherCapabilities
+
+        // Codecs not set in codecPreferences will not be negotiated in the offer
+        codecPreferences = combinedCapabilities.compactMap { $0 }
+
+        log("codecPreferences set: \(codecPreferences.map { String(describing: $0) }.joined(separator: ", "))")
+
+        assert(codecPreferences.first?.name.lowercased() == codec.id, "Preferred codec is not first in the list")
+    }
+}