fix(devices): API to disable speaking while muted notifications (#1335)

A joint PR that fixes a couple of related issues:
- Adds an API that disables speaking while muted notifications - fixes #1329
- The selected device can't be reliably detected when audio or video filters are used
- Audio tracks are now stopped (instead of disabled) on the Web - this should release the mic allocation
- Fixed an issue in `usePersistedDevicePreferences()` where the camera and mic were automatically enabled after a call had been left
- Pronto and Video Demo now remember the last used video filter and they enable it automatically

packages/client/docusaurus/docs/javascript/02-guides/04-camera-and-microphone.mdx

@@ -108,7 +108,7 @@ call.camera.selectDirection(defaultDirection);
 
 #### In call
 
-Follow our [Playing Video and Audio guide](../../guides/playing-video-and-audio/).
+Follow our [Rendering Video and Audio guide](../../guides/playing-video-and-audio/).
 
 #### Lobby preview
 
@@ -131,6 +131,20 @@ if (videoEl.srcObject !== call.camera.state.mediaStream) {
 }
 ```
 
+### Access to the Camera's MediaStream
+
+Our SDK exposes the current `mediaStream` instance that you can use for your needs (for example, local recording, etc...):
+
+```typescript
+// current mediaStream instance
+call.camera.state.mediaStream;
+// Reactive value for mediaStream, you can subscribe to changes
+call.camera.state.mediaStream$.subscribe((mediaStream) => {
+  const [videoTrack] = mediaStream.getVideoTracks();
+  console.log('Video track', videoTrack);
+});
+```
+
 ## Microphone management
 
 The SDK does its best to make working with the microphone easy. We expose the following objects on the call:
@@ -238,6 +252,7 @@ call.microphone.state.mediaStream$.subscribe(async (mediaStream) => {
 ### Speaking while muted notification
 
 When the microphone is disabled, the client will automatically start monitoring audio levels, to detect if the user is speaking.
+This feature is enabled by default unless the user doesn't have the permission to send audio or explicitly disabled.
 
 This is how you can subscribe to these notifications:
 
@@ -248,9 +263,27 @@ call.microphone.state.speakingWhileMuted$.subscribe((isSpeaking) => {
     console.log(`You're muted, unmute yourself to speak`);
   }
 }); // Reactive value
+
+// to disable this feature completely:
+await call.microphone.disableSpeakingWhileMutedNotification();
+
+// to enable it back:
+await call.microphone.enableSpeakingWhileMutedNotification();
 ```
 
-The notification is automatically disabled if the user doesn't have the permission to send audio.
+### Access to the Microphone's MediaStream
+
+Our SDK exposes the current `mediaStream` instance that you can use for your needs (for example, local recording, etc...):
+
+```typescript
+// current mediaStream instance
+call.microphone.state.mediaStream;
+// Reactive value for mediaStream, you can subscribe to changes
+call.microphone.state.mediaStream$.subscribe((mediaStream) => {
+  const [audioTrack] = mediaStream.getAudioTracks();
+  console.log('Audio track', audioTrack);
+});
+```
 
 ### Noise Cancellation
 

packages/client/src/devices/CameraManagerState.ts

@@ -45,8 +45,11 @@ export class CameraManagerState extends InputMediaDeviceManagerState {
   /**
    * @internal
    */
-  setMediaStream(stream: MediaStream | undefined): void {
-    super.setMediaStream(stream);
+  setMediaStream(
+    stream: MediaStream | undefined,
+    rootStream: MediaStream | undefined,
+  ): void {
+    super.setMediaStream(stream, rootStream);
     if (stream) {
       // RN getSettings() doesn't return facingMode, so we don't verify camera direction
       const direction = isReactNative()
@@ -59,8 +62,7 @@ export class CameraManagerState extends InputMediaDeviceManagerState {
   }
 
   protected getDeviceIdFromStream(stream: MediaStream): string | undefined {
-    return stream.getVideoTracks()[0]?.getSettings().deviceId as
-      | string
-      | undefined;
+    const [track] = stream.getVideoTracks();
+    return track?.getSettings().deviceId;
   }
 }

packages/client/src/devices/InputMediaDeviceManager.ts

@@ -209,7 +209,7 @@ export abstract class InputMediaDeviceManager<
         // @ts-expect-error called to dispose the stream in RN
         this.state.mediaStream.release();
       }
-      this.state.setMediaStream(undefined);
+      this.state.setMediaStream(undefined, undefined);
     }
   }
 
@@ -245,6 +245,7 @@ export abstract class InputMediaDeviceManager<
   protected async unmuteStream() {
     this.logger('debug', 'Starting stream');
     let stream: MediaStream;
+    let rootStream: Promise<MediaStream> | undefined;
     if (
       this.state.mediaStream &&
       this.getTracks().every((t) => t.readyState === 'live')
@@ -315,17 +316,20 @@ export abstract class InputMediaDeviceManager<
           return filterStream;
         };
 
+      // the rootStream represents the stream coming from the actual device
+      // e.g. camera or microphone stream
+      rootStream = this.getStream(constraints as C);
       // we publish the last MediaStream of the chain
       stream = await this.filters.reduce(
         (parent, filter) => parent.then(filter).then(chainWith(parent)),
-        this.getStream(constraints as C),
+        rootStream,
       );
     }
     if (this.call.state.callingState === CallingState.JOINED) {
       await this.publishStream(stream);
     }
     if (this.state.mediaStream !== stream) {
-      this.state.setMediaStream(stream);
+      this.state.setMediaStream(stream, await rootStream);
       this.getTracks().forEach((track) => {
         track.addEventListener('ended', async () => {
           if (this.enablePromise) {

packages/client/src/devices/InputMediaDeviceManagerState.ts

@@ -9,6 +9,7 @@ import { RxUtils } from '../store';
 import { getLogger } from '../logger';
 
 export type InputDeviceStatus = 'enabled' | 'disabled' | undefined;
+export type TrackDisableMode = 'stop-tracks' | 'disable-tracks';
 
 export abstract class InputMediaDeviceManagerState<C = MediaTrackConstraints> {
   protected statusSubject = new BehaviorSubject<InputDeviceStatus>(undefined);
@@ -102,9 +103,7 @@ export abstract class InputMediaDeviceManagerState<C = MediaTrackConstraints> {
    * `undefined` means no permission is required.
    */
   constructor(
-    public readonly disableMode:
-      | 'stop-tracks'
-      | 'disable-tracks' = 'stop-tracks',
+    public readonly disableMode: TrackDisableMode = 'stop-tracks',
     private readonly permissionName: PermissionName | undefined = undefined,
   ) {}
 
@@ -146,13 +145,20 @@ export abstract class InputMediaDeviceManagerState<C = MediaTrackConstraints> {
   }
 
   /**
+   * Updates the `mediaStream` state variable.
+   *
    * @internal
    * @param stream the stream to set.
+   * @param rootStream the root stream, applicable when filters are used
+   * as this is the stream that holds the actual deviceId information.
    */
-  setMediaStream(stream: MediaStream | undefined) {
+  setMediaStream(
+    stream: MediaStream | undefined,
+    rootStream: MediaStream | undefined,
+  ) {
     this.setCurrentValue(this.mediaStreamSubject, stream);
-    if (stream) {
-      this.setDevice(this.getDeviceIdFromStream(stream));
+    if (rootStream) {
+      this.setDevice(this.getDeviceIdFromStream(rootStream));
     }
   }
 

packages/client/src/devices/MicrophoneManager.ts

@@ -3,6 +3,7 @@ import type { INoiseCancellation } from '@stream-io/audio-filters-web';
 import { Call } from '../Call';
 import { InputMediaDeviceManager } from './InputMediaDeviceManager';
 import { MicrophoneManagerState } from './MicrophoneManagerState';
+import { TrackDisableMode } from './InputMediaDeviceManagerState';
 import { getAudioDevices, getAudioStream } from './devices';
 import { TrackType } from '../gen/video/sfu/models/models';
 import { createSoundDetector } from '../helpers/sound-detector';
@@ -16,37 +17,48 @@ import { createSubscription } from '../store/rxUtils';
 import { RNSpeechDetector } from '../helpers/RNSpeechDetector';
 
 export class MicrophoneManager extends InputMediaDeviceManager<MicrophoneManagerState> {
+  private speakingWhileMutedNotificationEnabled = true;
   private soundDetectorCleanup?: Function;
   private rnSpeechDetector: RNSpeechDetector | undefined;
   private noiseCancellation: INoiseCancellation | undefined;
   private noiseCancellationChangeUnsubscribe: (() => void) | undefined;
   private noiseCancellationRegistration?: Promise<() => Promise<void>>;
 
-  constructor(call: Call) {
-    super(call, new MicrophoneManagerState(), TrackType.AUDIO);
-
-    combineLatest([
-      this.call.state.callingState$,
-      this.call.state.ownCapabilities$,
-      this.state.selectedDevice$,
-      this.state.status$,
-    ]).subscribe(async ([callingState, ownCapabilities, deviceId, status]) => {
-      if (callingState !== CallingState.JOINED) {
-        if (callingState === CallingState.LEFT) {
-          await this.stopSpeakingWhileMutedDetection();
-        }
-        return;
-      }
-      if (ownCapabilities.includes(OwnCapability.SEND_AUDIO)) {
-        if (status === 'disabled') {
-          await this.startSpeakingWhileMutedDetection(deviceId);
-        } else {
-          await this.stopSpeakingWhileMutedDetection();
-        }
-      } else {
-        await this.stopSpeakingWhileMutedDetection();
-      }
-    });
+  constructor(
+    call: Call,
+    disableMode: TrackDisableMode = isReactNative()
+      ? 'disable-tracks'
+      : 'stop-tracks',
+  ) {
+    super(call, new MicrophoneManagerState(disableMode), TrackType.AUDIO);
+
+    this.subscriptions.push(
+      createSubscription(
+        combineLatest([
+          this.call.state.callingState$,
+          this.call.state.ownCapabilities$,
+          this.state.selectedDevice$,
+          this.state.status$,
+        ]),
+        async ([callingState, ownCapabilities, deviceId, status]) => {
+          if (callingState === CallingState.LEFT) {
+            await this.stopSpeakingWhileMutedDetection();
+          }
+          if (callingState !== CallingState.JOINED) return;
+          if (!this.speakingWhileMutedNotificationEnabled) return;
+
+          if (ownCapabilities.includes(OwnCapability.SEND_AUDIO)) {
+            if (status === 'disabled') {
+              await this.startSpeakingWhileMutedDetection(deviceId);
+            } else {
+              await this.stopSpeakingWhileMutedDetection();
+            }
+          } else {
+            await this.stopSpeakingWhileMutedDetection();
+          }
+        },
+      ),
+    );
 
     this.subscriptions.push(
       createSubscription(this.call.state.callingState$, (callingState) => {
@@ -163,6 +175,24 @@ export class MicrophoneManager extends InputMediaDeviceManager<MicrophoneManager
     await this.call.notifyNoiseCancellationStopped();
   }
 
+  /**
+   * Enables speaking while muted notification.
+   */
+  async enableSpeakingWhileMutedNotification() {
+    this.speakingWhileMutedNotificationEnabled = true;
+    if (this.state.status === 'disabled') {
+      await this.startSpeakingWhileMutedDetection(this.state.selectedDevice);
+    }
+  }
+
+  /**
+   * Disables speaking while muted notification.
+   */
+  async disableSpeakingWhileMutedNotification() {
+    this.speakingWhileMutedNotificationEnabled = false;
+    await this.stopSpeakingWhileMutedDetection();
+  }
+
   protected getDevices(): Observable<MediaDeviceInfo[]> {
     return getAudioDevices();
   }
@@ -208,9 +238,7 @@ export class MicrophoneManager extends InputMediaDeviceManager<MicrophoneManager
   }
 
   private async stopSpeakingWhileMutedDetection() {
-    if (!this.soundDetectorCleanup) {
-      return;
-    }
+    if (!this.soundDetectorCleanup) return;
     this.state.setSpeakingWhileMuted(false);
     try {
       await this.soundDetectorCleanup();

packages/client/src/devices/MicrophoneManagerState.ts

@@ -1,5 +1,8 @@
 import { BehaviorSubject, distinctUntilChanged, Observable } from 'rxjs';
-import { InputMediaDeviceManagerState } from './InputMediaDeviceManagerState';
+import {
+  InputMediaDeviceManagerState,
+  TrackDisableMode,
+} from './InputMediaDeviceManagerState';
 
 export class MicrophoneManagerState extends InputMediaDeviceManagerState {
   private speakingWhileMutedSubject = new BehaviorSubject<boolean>(false);
@@ -11,9 +14,9 @@ export class MicrophoneManagerState extends InputMediaDeviceManagerState {
    */
   speakingWhileMuted$: Observable<boolean>;
 
-  constructor() {
+  constructor(disableMode: TrackDisableMode) {
     super(
-      'disable-tracks',
+      disableMode,
       // `microphone` is not in the W3C standard yet,
       // but it's supported by Chrome and Safari.
       'microphone' as PermissionName,
@@ -41,8 +44,7 @@ export class MicrophoneManagerState extends InputMediaDeviceManagerState {
   }
 
   protected getDeviceIdFromStream(stream: MediaStream): string | undefined {
-    return stream.getAudioTracks()[0]?.getSettings().deviceId as
-      | string
-      | undefined;
+    const [track] = stream.getAudioTracks();
+    return track?.getSettings().deviceId;
   }
 }