fix(web): make rtc more stable (#937)

* fix(web): listen to track-ended

* refactor(web): add hot plug

* refactor(web): add check of low volume

* refactor(web): adjust low volume value, add more logs

* refactor(web): reduce logs & only check local volume

packages/flat-i18n/locales/en.json

@@ -351,6 +351,7 @@
   "servers": "Server",
   "set-camera-error": "Failed to turn on camera",
   "set-mic-error": "Failed to turn on microphone",
+  "low-volume": "The volume is too low",
   "share-screen": {
     "browser-not-permission": "Please grant your browser access to screen recording",
     "desktop-not-permission": "Please grant Flat access to screen recording",

packages/flat-i18n/locales/zh-CN.json

@@ -351,6 +351,7 @@
   "servers": "服务器",
   "set-camera-error": "无法检测到摄像头，请检查您的设备后重试",
   "set-mic-error": "无法检测到麦克风，请检查您的设备后重试",
+  "low-volume": "音量过低",
   "share-screen": {
     "browser-not-permission": "请授予浏览器访问屏幕录制的权限",
     "desktop-not-permission": "请授予 Flat 访问屏幕录制的权限",

web/flat-web/src/apiMiddleware/rtc/avatar.ts

@@ -2,7 +2,9 @@ import type {
     IAgoraRTCClient,
     IAgoraRTCRemoteUser,
     ICameraVideoTrack,
+    ILocalAudioTrack,
     IMicrophoneAudioTrack,
+    IRemoteAudioTrack,
     ITrack,
 } from "agora-rtc-sdk-ng";
 import { EventEmitter } from "eventemitter3";
@@ -18,6 +20,7 @@ export interface RtcAvatarParams {
 export enum RtcEvents {
     SetCameraError = "set-camera-error",
     SetMicError = "set-mic-error",
+    LowVolume = "low-volume",
 }
 
 /**
@@ -27,6 +30,9 @@ export enum RtcEvents {
  * avatar.setCamera(true)
  */
 export class RtcAvatar extends EventEmitter {
+    public static readonly LowVolume = 0.1;
+    public static readonly LowVolumeMaxCount = 10;
+
     public readonly userUUID: string;
     public readonly avatarUser: User;
     public element?: HTMLElement;
@@ -38,6 +44,8 @@ export class RtcAvatar extends EventEmitter {
     private remoteUser?: IAgoraRTCRemoteUser;
     private mic = false;
     private camera = false;
+    private observeVolumeId: number;
+    private observeVolumeCounter = 0;
 
     public constructor({ rtc, userUUID, avatarUser }: RtcAvatarParams) {
         super();
@@ -46,9 +54,11 @@ export class RtcAvatar extends EventEmitter {
         this.avatarUser = avatarUser;
         this.isLocal = userUUID === avatarUser.userUUID;
         this.rtc.addAvatar(this);
+        this.observeVolumeId = window.setInterval(this.checkVolume, 500);
     }
 
     public destroy(): void {
+        clearInterval(this.observeVolumeId);
         this.rtc.removeAvatar(this);
     }
 
@@ -93,31 +103,57 @@ export class RtcAvatar extends EventEmitter {
     }
 
     private async refreshLocalCamera(): Promise<void> {
-        if (this.camera && !this.videoTrack) {
-            this.videoTrack = await this.rtc.getLocalVideoTrack();
-            this.element && this.videoTrack.play(this.element);
-        } else if (this.videoTrack && this.videoTrack.isPlaying !== this.camera) {
-            await (this.videoTrack as ICameraVideoTrack).setEnabled(this.camera);
+        try {
+            if (this.camera && !this.videoTrack) {
+                this.videoTrack = await this.rtc.getLocalVideoTrack();
+                this.element && this.videoTrack?.play(this.element);
+            } else if (this.videoTrack && this.videoTrack.isPlaying !== this.camera) {
+                await (this.videoTrack as ICameraVideoTrack).setEnabled(this.camera);
+            }
+        } catch (error) {
+            this.videoTrack = undefined;
+            this.emit(RtcEvents.SetCameraError, error);
         }
     }
 
     private async refreshLocalMic(): Promise<void> {
-        if (this.mic && !this.audioTrack) {
-            this.audioTrack = await this.rtc.getLocalAudioTrack();
-            // NOTE: play local audio will cause echo
-            // this.audioTrack.play();
-        } else if (this.audioTrack && this.audioTrack.isPlaying !== this.mic) {
-            await (this.audioTrack as IMicrophoneAudioTrack).setEnabled(this.mic);
+        try {
+            if (this.mic && !this.audioTrack) {
+                this.audioTrack = await this.rtc.getLocalAudioTrack();
+                // NOTE: play local audio will cause echo
+                // this.audioTrack.play();
+            } else if (this.audioTrack && this.audioTrack.isPlaying !== this.mic) {
+                await (this.audioTrack as IMicrophoneAudioTrack).setEnabled(this.mic);
+            }
+        } catch (error) {
+            this.audioTrack = undefined;
+            this.emit(RtcEvents.SetMicError, error);
         }
     }
 
     public async setCamera(enable: boolean): Promise<void> {
         this.camera = enable;
-        await this.refresh().catch(error => this.emit(RtcEvents.SetMicError, error));
+        await this.refresh().catch(error => this.emit(RtcEvents.SetCameraError, error));
     }
 
     public async setMic(enable: boolean): Promise<void> {
         this.mic = enable;
-        await this.refresh().catch(error => this.emit(RtcEvents.SetCameraError, error));
+        await this.refresh().catch(error => this.emit(RtcEvents.SetMicError, error));
     }
+
+    private checkVolume = (): void => {
+        if (this.isLocal && this.mic && this.audioTrack) {
+            const track = this.audioTrack as ILocalAudioTrack | IRemoteAudioTrack;
+            const volume = track.getVolumeLevel();
+            if (volume < RtcAvatar.LowVolume) {
+                console.log("[rtc] volume low: %O", volume);
+                this.observeVolumeCounter += 1;
+                if (this.observeVolumeCounter === RtcAvatar.LowVolumeMaxCount) {
+                    this.emit(RtcEvents.LowVolume);
+                }
+            }
+        } else {
+            this.observeVolumeCounter = 0;
+        }
+    };
 }

web/flat-web/src/apiMiddleware/rtc/hot-plug.ts

@@ -0,0 +1,36 @@
+import AgoraRTC, { ICameraVideoTrack } from "agora-rtc-sdk-ng";
+import type { IMicrophoneAudioTrack } from "agora-rtc-sdk-ng";
+
+let microphoneTrack: IMicrophoneAudioTrack | undefined;
+export function setMicrophoneTrack(track?: IMicrophoneAudioTrack): void {
+    microphoneTrack = track;
+}
+
+let cameraTrack: ICameraVideoTrack | undefined;
+export function setCameraTrack(track?: ICameraVideoTrack): void {
+    cameraTrack = track;
+}
+
+AgoraRTC.onMicrophoneChanged = async changedDevice => {
+    if (changedDevice.state === "ACTIVE") {
+        console.log("[rtc] microphone new device: %s", changedDevice.device.deviceId);
+        microphoneTrack?.setDevice(changedDevice.device.deviceId);
+    } else if (changedDevice.device.label === microphoneTrack?.getTrackLabel()) {
+        console.log("[rtc] microphone pull out");
+        const [microphone] = await AgoraRTC.getMicrophones();
+        console.log("[rtc] microphone old device: %s", microphone.deviceId);
+        microphone && microphoneTrack.setDevice(microphone.deviceId);
+    }
+};
+
+AgoraRTC.onCameraChanged = async changedDevice => {
+    if (changedDevice.state === "ACTIVE") {
+        console.log("[rtc] camera new device: %s", changedDevice.device.deviceId);
+        cameraTrack?.setDevice(changedDevice.device.deviceId);
+    } else if (changedDevice.device.label === cameraTrack?.getTrackLabel()) {
+        console.log("[rtc] camera pull out");
+        const [camera] = await AgoraRTC.getMicrophones();
+        console.log("[rtc] camera old device: %s", camera.deviceId);
+        camera && cameraTrack.setDevice(camera.deviceId);
+    }
+};

web/flat-web/src/apiMiddleware/rtc/room.ts

@@ -1,14 +1,17 @@
 import type {
     IAgoraRTCClient,
     IAgoraRTCRemoteUser,
+    ICameraVideoTrack,
     ILocalAudioTrack,
     ILocalVideoTrack,
+    IMicrophoneAudioTrack,
 } from "agora-rtc-sdk-ng";
 import AgoraRTC from "agora-rtc-sdk-ng";
 import type { RtcAvatar } from "./avatar";
 import { AGORA } from "../../constants/Process";
 import { globalStore } from "../../stores/GlobalStore";
 import { generateRTCToken } from "../flatServer/agora";
+import { setCameraTrack, setMicrophoneTrack } from "./hot-plug";
 
 AgoraRTC.enableLogUpload();
 
@@ -76,11 +79,14 @@ export class RtcRoom {
 
     public async destroy(): Promise<void> {
         if (this.client) {
+            setMicrophoneTrack();
+            setCameraTrack();
             if (this.client.localTracks.length > 0) {
                 for (const track of this.client.localTracks) {
                     track.stop();
                     track.close();
                 }
+                console.log("[rtc] unpublish local tracks");
                 await this.client.unpublish(this.client.localTracks);
             }
             this.client.off("user-published", this.onUserPublished);
@@ -106,6 +112,11 @@ export class RtcRoom {
     public async getLocalAudioTrack(): Promise<ILocalAudioTrack> {
         if (!this._localAudioTrack) {
             this._localAudioTrack = await AgoraRTC.createMicrophoneAudioTrack();
+            setMicrophoneTrack(this._localAudioTrack as IMicrophoneAudioTrack);
+            this._localAudioTrack.once("track-ended", () => {
+                console.log("[rtc] track-ended local audio");
+            });
+            console.log("[rtc] publish audio track");
             await this.client?.publish(this._localAudioTrack);
         }
         return this._localAudioTrack;
@@ -116,6 +127,11 @@ export class RtcRoom {
             this._localVideoTrack = await AgoraRTC.createCameraVideoTrack({
                 encoderConfig: { width: 288, height: 216 },
             });
+            setCameraTrack(this._localVideoTrack as ICameraVideoTrack);
+            this._localVideoTrack.once("track-ended", () => {
+                console.log("[rtc] track-ended local video");
+            });
+            console.log("[rtc] publish video track");
             await this.client?.publish(this._localVideoTrack);
         }
         return this._localVideoTrack;

web/flat-web/src/components/AvatarCanvas.tsx

@@ -37,13 +37,17 @@ export const AvatarCanvas = observer<AvatarCanvasProps>(function AvatarCanvas({
 
     useEffect(() => {
         rtcAvatar.on(RtcEvents.SetCameraError, (error: Error) => {
-            console.log("set camera error", error);
+            console.log("[rtc] set camera error", error);
             void message.error(t("set-camera-error"));
         });
         rtcAvatar.on(RtcEvents.SetMicError, (error: Error) => {
-            console.log("set microphone error", error);
+            console.log("[rtc] set microphone error", error);
             void message.error(t("set-mic-error"));
         });
+        rtcAvatar.on(RtcEvents.LowVolume, () => {
+            console.log("[rtc] low volume");
+            void message.warn(t("low-volume"));
+        });
         return () => void rtcAvatar.destroy();
     }, [rtcAvatar, t]);