Fix VirtualBackgroundVideoProcessor not responding to changing backgroundImage (#752)

* Fix VirtualBackgroundVideoProcessor not responding to changes in backgroundImage

* lint fixes

* spotless

Author: davidliu

.changeset/olive-bottles-repeat.md

@@ -0,0 +1,5 @@
+---
+"client-sdk-android": patch
+---
+
+Fix VirtualBackgroundVideoProcessor not responding to changes in backgroundImage

examples/virtual-background/src/main/java/io/livekit/android/selfie/MainActivity.kt

@@ -45,6 +45,17 @@ class MainActivity : AppCompatActivity() {
             enableButton.setText(if (state) "Disable" else "Enable")
         }
 
+        val enableBackgroundButton = findViewById<Button>(R.id.buttonBackground)
+        enableBackgroundButton.setOnClickListener {
+            val state = viewModel.toggleVirtualBackground()
+            enableBackgroundButton.setText(if (state) "Disable Background" else "Enable Background")
+        }
+
+        val flipCameraButton = findViewById<Button>(R.id.buttonFlip)
+        flipCameraButton.setOnClickListener {
+            viewModel.flipCamera()
+        }
+
         val renderer = findViewById<TextureViewRenderer>(R.id.renderer)
         viewModel.room.initVideoRenderer(renderer)
         viewModel.track.observe(this) { track ->
@@ -87,7 +98,7 @@ fun ComponentActivity.requestNeededPermissions(onPermissionsGranted: (() -> Unit
             }
         }
 
-    val neededPermissions = listOf(Manifest.permission.RECORD_AUDIO, Manifest.permission.CAMERA)
+    val neededPermissions = listOf(Manifest.permission.CAMERA)
         .filter { ContextCompat.checkSelfPermission(this, it) == PackageManager.PERMISSION_DENIED }
         .toTypedArray()
 

examples/virtual-background/src/main/java/io/livekit/android/selfie/MainViewModel.kt

@@ -54,10 +54,12 @@ class MainViewModel(application: Application) : AndroidViewModel(application) {
             eglBase = eglBase,
         ),
     )
+
+    private val virtualBackground = (AppCompatResources.getDrawable(application, R.drawable.background) as BitmapDrawable).bitmap
+
     private var blur = 16f
     private val processor = VirtualBackgroundVideoProcessor(eglBase, Dispatchers.IO, initialBlurRadius = blur).apply {
-        val drawable = AppCompatResources.getDrawable(application, R.drawable.background) as BitmapDrawable
-        backgroundImage = drawable.bitmap
+        backgroundImage = virtualBackground
     }
 
     private var cameraProvider: CameraCapturerUtils.CameraProvider? = null
@@ -119,4 +121,24 @@ class MainViewModel(application: Application) : AndroidViewModel(application) {
         blur += 5
         processor.updateBlurRadius(blur)
     }
+
+    fun toggleVirtualBackground(): Boolean {
+        if (processor.backgroundImage != virtualBackground) {
+            processor.backgroundImage = virtualBackground
+            return true
+        } else {
+            processor.backgroundImage = null
+            return false
+        }
+    }
+
+    fun flipCamera() {
+        val videoTrack = track.value ?: return
+        val newPosition = when (videoTrack.options.position) {
+            CameraPosition.FRONT -> CameraPosition.BACK
+            CameraPosition.BACK -> CameraPosition.FRONT
+            else -> CameraPosition.FRONT
+        }
+        videoTrack.switchCamera(position = newPosition)
+    }
 }

examples/virtual-background/src/main/res/layout/activity_main.xml

@@ -10,12 +10,32 @@
         android:layout_width="match_parent"
         android:layout_height="match_parent" />
 
-    <Button
-        android:id="@+id/button"
+    <LinearLayout
         android:layout_width="wrap_content"
         android:layout_height="wrap_content"
-        android:layout_margin="10dp"
-        android:text="Disable" />
+        android:orientation="vertical">
+
+        <Button
+            android:id="@+id/button"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:layout_margin="10dp"
+            android:text="Disable" />
+
+        <Button
+            android:id="@+id/buttonBackground"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:layout_margin="10dp"
+            android:text="Disable Background" />
+
+        <Button
+            android:id="@+id/buttonFlip"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:layout_margin="10dp"
+            android:text="Flip Camera" />
+    </LinearLayout>
 
     <Button
         android:id="@+id/buttonIncreaseBlur"

livekit-android-camerax/src/main/java/livekit/org/webrtc/CameraXSession.kt

@@ -26,7 +26,6 @@ import android.hardware.camera2.CameraMetadata.LENS_OPTICAL_STABILIZATION_MODE_O
 import android.hardware.camera2.CameraMetadata.LENS_OPTICAL_STABILIZATION_MODE_ON
 import android.hardware.camera2.CaptureRequest
 import android.os.Build
-import android.os.Build.VERSION
 import android.os.Handler
 import android.util.Range
 import android.util.Size
@@ -345,7 +344,7 @@ internal constructor(
             if (id == deviceId) return CameraDeviceId(id, null)
 
             // Then check if deviceId is a physical camera ID in a logical camera
-            if (VERSION.SDK_INT >= Build.VERSION_CODES.P) {
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
                 val characteristic = cameraManager.getCameraCharacteristics(id)
 
                 for (physicalId in characteristic.physicalCameraIds) {

livekit-android-sdk/src/main/java/io/livekit/android/audio/AudioSwitchHandler.kt

@@ -23,7 +23,12 @@ import android.os.Build
 import android.os.Handler
 import android.os.HandlerThread
 import android.os.Looper
-import com.twilio.audioswitch.*
+import com.twilio.audioswitch.AbstractAudioSwitch
+import com.twilio.audioswitch.AudioDevice
+import com.twilio.audioswitch.AudioDeviceChangeListener
+import com.twilio.audioswitch.AudioSwitch
+import com.twilio.audioswitch.LegacyAudioSwitch
+import io.livekit.android.room.Room
 import io.livekit.android.util.LKLog
 import javax.inject.Inject
 import javax.inject.Singleton

livekit-android-sdk/src/main/java/io/livekit/android/audio/ScreenAudioCapturer.kt

@@ -234,7 +234,6 @@ constructor(
             AudioFormat.ENCODING_PCM_8BIT -> 1
             AudioFormat.ENCODING_PCM_16BIT, AudioFormat.ENCODING_IEC61937, AudioFormat.ENCODING_DEFAULT -> 2
             AudioFormat.ENCODING_PCM_FLOAT -> 4
-            AudioFormat.ENCODING_INVALID -> throw IllegalArgumentException("Bad audio format $audioFormat")
             else -> throw IllegalArgumentException("Bad audio format $audioFormat")
         }
     }

livekit-android-sdk/src/main/java/io/livekit/android/e2ee/E2EEManager.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023-2024 LiveKit, Inc.
+ * Copyright 2023-2025 LiveKit, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -58,7 +58,7 @@ constructor(
         this.peerConnectionFactory = peerConnectionFactory
     }
 
-    public fun keyProvider(): KeyProvider {
+    fun keyProvider(): KeyProvider {
         return this.keyProvider
     }
 
@@ -70,16 +70,16 @@ constructor(
         this.enabled = true
         this.room = room
         this.emitEvent = emitEvent
-        this.room?.localParticipant?.trackPublications?.forEach() { item ->
+        this.room?.localParticipant?.trackPublications?.forEach { item ->
             var participant = this.room!!.localParticipant
             var publication = item.value
             if (publication.track != null) {
                 addPublishedTrack(publication.track!!, publication, participant, room)
             }
         }
-        this.room?.remoteParticipants?.forEach() { item ->
+        this.room?.remoteParticipants?.forEach { item ->
             var participant = item.value
-            participant.trackPublications.forEach() { item ->
+            participant.trackPublications.forEach { item ->
                 var publication = item.value
                 if (publication.track != null) {
                     addSubscribedTrack(publication.track!!, publication, participant, room)
@@ -88,7 +88,7 @@ constructor(
         }
     }
 
-    public fun addSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room) {
+    fun addSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room) {
         var rtpReceiver: RtpReceiver? = when (publication.track!!) {
             is RemoteAudioTrack -> (publication.track!! as RemoteAudioTrack).receiver
             is RemoteVideoTrack -> (publication.track!! as RemoteVideoTrack).receiver
@@ -111,7 +111,7 @@ constructor(
         }
     }
 
-    public fun removeSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room) {
+    fun removeSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room) {
         var trackId = publication.sid
         var participantId = participant.identity
         var frameCryptor = frameCryptors.get(trackId to participantId)
@@ -122,7 +122,7 @@ constructor(
         }
     }
 
-    public fun addPublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room) {
+    fun addPublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room) {
         var rtpSender: RtpSender? = when (publication.track!!) {
             is LocalAudioTrack -> (publication.track!! as LocalAudioTrack)?.sender
             is LocalVideoTrack -> (publication.track!! as LocalVideoTrack)?.sender
@@ -146,7 +146,7 @@ constructor(
         }
     }
 
-    public fun removePublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room) {
+    fun removePublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room) {
         var trackId = publication.sid
         var participantId = participant.identity
         var frameCryptor = frameCryptors.get(trackId to participantId)
@@ -202,7 +202,7 @@ constructor(
      * Enable or disable E2EE
      * @param enabled
      */
-    public fun enableE2EE(enabled: Boolean) {
+    fun enableE2EE(enabled: Boolean) {
         this.enabled = enabled
         for (item in frameCryptors.entries) {
             var frameCryptor = item.value

livekit-android-sdk/src/main/java/io/livekit/android/e2ee/E2EEOptions.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023-2024 LiveKit, Inc.
+ * Copyright 2023-2025 LiveKit, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,9 +25,8 @@ internal const val defaultFailureTolerance = -1
 internal const val defaultKeyRingSize = 16
 internal const val defaultDiscardFrameWhenCryptorNotReady = false
 
-class E2EEOptions
-constructor(
-    keyProvider: KeyProvider = BaseKeyProvider(
+class E2EEOptions(
+    var keyProvider: KeyProvider = BaseKeyProvider(
         defaultRatchetSalt,
         defaultMagicBytes,
         defaultRatchetWindowSize,
@@ -38,11 +37,9 @@ constructor(
     ),
     encryptionType: Encryption.Type = Encryption.Type.GCM,
 ) {
-    var keyProvider: KeyProvider
     var encryptionType: Encryption.Type = Encryption.Type.NONE
 
     init {
-        this.keyProvider = keyProvider
         this.encryptionType = encryptionType
     }
 }

livekit-android-sdk/src/main/java/io/livekit/android/e2ee/KeyProvider.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023-2024 LiveKit, Inc.
+ * Copyright 2023-2025 LiveKit, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,14 +20,13 @@ import io.livekit.android.util.LKLog
 import livekit.org.webrtc.FrameCryptorFactory
 import livekit.org.webrtc.FrameCryptorKeyProvider
 
-class KeyInfo
-constructor(var participantId: String, var keyIndex: Int, var key: String) {
+class KeyInfo(var participantId: String, var keyIndex: Int, var key: String) {
     override fun toString(): String {
         return "KeyInfo(participantId='$participantId', keyIndex=$keyIndex)"
     }
 }
 
-public interface KeyProvider {
+interface KeyProvider {
     fun setSharedKey(key: String, keyIndex: Int? = 0): Boolean
     fun ratchetSharedKey(keyIndex: Int? = 0): ByteArray
     fun exportSharedKey(keyIndex: Int? = 0): ByteArray
@@ -41,17 +40,30 @@ public interface KeyProvider {
     var enableSharedKey: Boolean
 }
 
-class BaseKeyProvider
-constructor(
-    private var ratchetSalt: String = defaultRatchetSalt,
-    private var uncryptedMagicBytes: String = defaultMagicBytes,
-    private var ratchetWindowSize: Int = defaultRatchetWindowSize,
+class BaseKeyProvider(
+    ratchetSalt: String = defaultRatchetSalt,
+    uncryptedMagicBytes: String = defaultMagicBytes,
+    ratchetWindowSize: Int = defaultRatchetWindowSize,
     override var enableSharedKey: Boolean = true,
-    private var failureTolerance: Int = defaultFailureTolerance,
-    private var keyRingSize: Int = defaultKeyRingSize,
-    private var discardFrameWhenCryptorNotReady: Boolean = defaultDiscardFrameWhenCryptorNotReady,
+    failureTolerance: Int = defaultFailureTolerance,
+    keyRingSize: Int = defaultKeyRingSize,
+    discardFrameWhenCryptorNotReady: Boolean = defaultDiscardFrameWhenCryptorNotReady,
 ) : KeyProvider {
+    override val rtcKeyProvider: FrameCryptorKeyProvider
+
     private var keys: MutableMap<String, MutableMap<Int, String>> = mutableMapOf()
+
+    init {
+        this.rtcKeyProvider = FrameCryptorFactory.createFrameCryptorKeyProvider(
+            enableSharedKey,
+            ratchetSalt.toByteArray(),
+            ratchetWindowSize,
+            uncryptedMagicBytes.toByteArray(),
+            failureTolerance,
+            keyRingSize,
+            discardFrameWhenCryptorNotReady,
+        )
+    }
     override fun setSharedKey(key: String, keyIndex: Int?): Boolean {
         return rtcKeyProvider.setSharedKey(keyIndex ?: 0, key.toByteArray())
     }
@@ -100,18 +112,4 @@ constructor(
     override fun setSifTrailer(trailer: ByteArray) {
         rtcKeyProvider.setSifTrailer(trailer)
     }
-
-    override val rtcKeyProvider: FrameCryptorKeyProvider
-
-    init {
-        this.rtcKeyProvider = FrameCryptorFactory.createFrameCryptorKeyProvider(
-            enableSharedKey,
-            ratchetSalt.toByteArray(),
-            ratchetWindowSize,
-            uncryptedMagicBytes.toByteArray(),
-            failureTolerance,
-            keyRingSize,
-            discardFrameWhenCryptorNotReady,
-        )
-    }
 }