diff --git a/packages/flat-i18n/locales/en.json b/packages/flat-i18n/locales/en.json index fbd5f1ba07a..50002a02aa8 100644 --- a/packages/flat-i18n/locales/en.json +++ b/packages/flat-i18n/locales/en.json @@ -351,6 +351,7 @@ "servers": "Server", "set-camera-error": "Failed to turn on camera", "set-mic-error": "Failed to turn on microphone", + "low-volume": "The volume is too low", "share-screen": { "browser-not-permission": "Please grant your browser access to screen recording", "desktop-not-permission": "Please grant Flat access to screen recording", diff --git a/packages/flat-i18n/locales/zh-CN.json b/packages/flat-i18n/locales/zh-CN.json index 9ffb000a075..c58eb1450b4 100644 --- a/packages/flat-i18n/locales/zh-CN.json +++ b/packages/flat-i18n/locales/zh-CN.json @@ -351,6 +351,7 @@ "servers": "服务器", "set-camera-error": "无法检测到摄像头，请检查您的设备后重试", "set-mic-error": "无法检测到麦克风，请检查您的设备后重试", + "low-volume": "音量过低", "share-screen": { "browser-not-permission": "请授予浏览器访问屏幕录制的权限", "desktop-not-permission": "请授予 Flat 访问屏幕录制的权限", diff --git a/web/flat-web/src/apiMiddleware/rtc/avatar.ts b/web/flat-web/src/apiMiddleware/rtc/avatar.ts index 0ff6ccee51d..362f06cd513 100644 --- a/web/flat-web/src/apiMiddleware/rtc/avatar.ts +++ b/web/flat-web/src/apiMiddleware/rtc/avatar.ts @@ -2,7 +2,9 @@ import type { IAgoraRTCClient, IAgoraRTCRemoteUser, ICameraVideoTrack, + ILocalAudioTrack, IMicrophoneAudioTrack, + IRemoteAudioTrack, ITrack, } from "agora-rtc-sdk-ng"; import { EventEmitter } from "eventemitter3"; @@ -18,6 +20,7 @@ export interface RtcAvatarParams { export enum RtcEvents { SetCameraError = "set-camera-error", SetMicError = "set-mic-error", + LowVolume = "low-volume", } /** @@ -27,6 +30,9 @@ export enum RtcEvents { * avatar.setCamera(true) */ export class RtcAvatar extends EventEmitter { + public static readonly LowVolume = 0.1; + public static readonly LowVolumeMaxCount = 10; + public readonly userUUID: string; public readonly avatarUser: User; public element?: HTMLElement; @@ -38,6 +44,8 @@ export class RtcAvatar extends EventEmitter { private remoteUser?: IAgoraRTCRemoteUser; private mic = false; private camera = false; + private observeVolumeId: number; + private observeVolumeCounter = 0; public constructor({ rtc, userUUID, avatarUser }: RtcAvatarParams) { super(); @@ -46,9 +54,11 @@ export class RtcAvatar extends EventEmitter { this.avatarUser = avatarUser; this.isLocal = userUUID === avatarUser.userUUID; this.rtc.addAvatar(this); + this.observeVolumeId = window.setInterval(this.checkVolume, 500); } public destroy(): void { + clearInterval(this.observeVolumeId); this.rtc.removeAvatar(this); } @@ -93,31 +103,57 @@ export class RtcAvatar extends EventEmitter { } private async refreshLocalCamera(): Promise { - if (this.camera && !this.videoTrack) { - this.videoTrack = await this.rtc.getLocalVideoTrack(); - this.element && this.videoTrack.play(this.element); - } else if (this.videoTrack && this.videoTrack.isPlaying !== this.camera) { - await (this.videoTrack as ICameraVideoTrack).setEnabled(this.camera); + try { + if (this.camera && !this.videoTrack) { + this.videoTrack = await this.rtc.getLocalVideoTrack(); + this.element && this.videoTrack?.play(this.element); + } else if (this.videoTrack && this.videoTrack.isPlaying !== this.camera) { + await (this.videoTrack as ICameraVideoTrack).setEnabled(this.camera); + } + } catch (error) { + this.videoTrack = undefined; + this.emit(RtcEvents.SetCameraError, error); } } private async refreshLocalMic(): Promise { - if (this.mic && !this.audioTrack) { - this.audioTrack = await this.rtc.getLocalAudioTrack(); - // NOTE: play local audio will cause echo - // this.audioTrack.play(); - } else if (this.audioTrack && this.audioTrack.isPlaying !== this.mic) { - await (this.audioTrack as IMicrophoneAudioTrack).setEnabled(this.mic); + try { + if (this.mic && !this.audioTrack) { + this.audioTrack = await this.rtc.getLocalAudioTrack(); + // NOTE: play local audio will cause echo + // this.audioTrack.play(); + } else if (this.audioTrack && this.audioTrack.isPlaying !== this.mic) { + await (this.audioTrack as IMicrophoneAudioTrack).setEnabled(this.mic); + } + } catch (error) { + this.audioTrack = undefined; + this.emit(RtcEvents.SetMicError, error); } } public async setCamera(enable: boolean): Promise { this.camera = enable; - await this.refresh().catch(error => this.emit(RtcEvents.SetMicError, error)); + await this.refresh().catch(error => this.emit(RtcEvents.SetCameraError, error)); } public async setMic(enable: boolean): Promise { this.mic = enable; - await this.refresh().catch(error => this.emit(RtcEvents.SetCameraError, error)); + await this.refresh().catch(error => this.emit(RtcEvents.SetMicError, error)); } + + private checkVolume = (): void => { + if (this.isLocal && this.mic && this.audioTrack) { + const track = this.audioTrack as ILocalAudioTrack | IRemoteAudioTrack; + const volume = track.getVolumeLevel(); + if (volume < RtcAvatar.LowVolume) { + console.log("[rtc] volume low: %O", volume); + this.observeVolumeCounter += 1; + if (this.observeVolumeCounter === RtcAvatar.LowVolumeMaxCount) { + this.emit(RtcEvents.LowVolume); + } + } + } else { + this.observeVolumeCounter = 0; + } + }; } diff --git a/web/flat-web/src/apiMiddleware/rtc/hot-plug.ts b/web/flat-web/src/apiMiddleware/rtc/hot-plug.ts new file mode 100644 index 00000000000..07c41b2a3aa --- /dev/null +++ b/web/flat-web/src/apiMiddleware/rtc/hot-plug.ts @@ -0,0 +1,36 @@ +import AgoraRTC, { ICameraVideoTrack } from "agora-rtc-sdk-ng"; +import type { IMicrophoneAudioTrack } from "agora-rtc-sdk-ng"; + +let microphoneTrack: IMicrophoneAudioTrack | undefined; +export function setMicrophoneTrack(track?: IMicrophoneAudioTrack): void { + microphoneTrack = track; +} + +let cameraTrack: ICameraVideoTrack | undefined; +export function setCameraTrack(track?: ICameraVideoTrack): void { + cameraTrack = track; +} + +AgoraRTC.onMicrophoneChanged = async changedDevice => { + if (changedDevice.state === "ACTIVE") { + console.log("[rtc] microphone new device: %s", changedDevice.device.deviceId); + microphoneTrack?.setDevice(changedDevice.device.deviceId); + } else if (changedDevice.device.label === microphoneTrack?.getTrackLabel()) { + console.log("[rtc] microphone pull out"); + const [microphone] = await AgoraRTC.getMicrophones(); + console.log("[rtc] microphone old device: %s", microphone.deviceId); + microphone && microphoneTrack.setDevice(microphone.deviceId); + } +}; + +AgoraRTC.onCameraChanged = async changedDevice => { + if (changedDevice.state === "ACTIVE") { + console.log("[rtc] camera new device: %s", changedDevice.device.deviceId); + cameraTrack?.setDevice(changedDevice.device.deviceId); + } else if (changedDevice.device.label === cameraTrack?.getTrackLabel()) { + console.log("[rtc] camera pull out"); + const [camera] = await AgoraRTC.getMicrophones(); + console.log("[rtc] camera old device: %s", camera.deviceId); + camera && cameraTrack.setDevice(camera.deviceId); + } +}; diff --git a/web/flat-web/src/apiMiddleware/rtc/room.ts b/web/flat-web/src/apiMiddleware/rtc/room.ts index ebb28a54af7..2da863c0d8b 100644 --- a/web/flat-web/src/apiMiddleware/rtc/room.ts +++ b/web/flat-web/src/apiMiddleware/rtc/room.ts @@ -1,14 +1,17 @@ import type { IAgoraRTCClient, IAgoraRTCRemoteUser, + ICameraVideoTrack, ILocalAudioTrack, ILocalVideoTrack, + IMicrophoneAudioTrack, } from "agora-rtc-sdk-ng"; import AgoraRTC from "agora-rtc-sdk-ng"; import type { RtcAvatar } from "./avatar"; import { AGORA } from "../../constants/Process"; import { globalStore } from "../../stores/GlobalStore"; import { generateRTCToken } from "../flatServer/agora"; +import { setCameraTrack, setMicrophoneTrack } from "./hot-plug"; AgoraRTC.enableLogUpload(); @@ -76,11 +79,14 @@ export class RtcRoom { public async destroy(): Promise { if (this.client) { + setMicrophoneTrack(); + setCameraTrack(); if (this.client.localTracks.length > 0) { for (const track of this.client.localTracks) { track.stop(); track.close(); } + console.log("[rtc] unpublish local tracks"); await this.client.unpublish(this.client.localTracks); } this.client.off("user-published", this.onUserPublished); @@ -106,6 +112,11 @@ export class RtcRoom { public async getLocalAudioTrack(): Promise { if (!this._localAudioTrack) { this._localAudioTrack = await AgoraRTC.createMicrophoneAudioTrack(); + setMicrophoneTrack(this._localAudioTrack as IMicrophoneAudioTrack); + this._localAudioTrack.once("track-ended", () => { + console.log("[rtc] track-ended local audio"); + }); + console.log("[rtc] publish audio track"); await this.client?.publish(this._localAudioTrack); } return this._localAudioTrack; @@ -116,6 +127,11 @@ export class RtcRoom { this._localVideoTrack = await AgoraRTC.createCameraVideoTrack({ encoderConfig: { width: 288, height: 216 }, }); + setCameraTrack(this._localVideoTrack as ICameraVideoTrack); + this._localVideoTrack.once("track-ended", () => { + console.log("[rtc] track-ended local video"); + }); + console.log("[rtc] publish video track"); await this.client?.publish(this._localVideoTrack); } return this._localVideoTrack; diff --git a/web/flat-web/src/components/AvatarCanvas.tsx b/web/flat-web/src/components/AvatarCanvas.tsx index a73f15d7d24..9c4ef4a7522 100644 --- a/web/flat-web/src/components/AvatarCanvas.tsx +++ b/web/flat-web/src/components/AvatarCanvas.tsx @@ -37,13 +37,17 @@ export const AvatarCanvas = observer(function AvatarCanvas({ useEffect(() => { rtcAvatar.on(RtcEvents.SetCameraError, (error: Error) => { - console.log("set camera error", error); + console.log("[rtc] set camera error", error); void message.error(t("set-camera-error")); }); rtcAvatar.on(RtcEvents.SetMicError, (error: Error) => { - console.log("set microphone error", error); + console.log("[rtc] set microphone error", error); void message.error(t("set-mic-error")); }); + rtcAvatar.on(RtcEvents.LowVolume, () => { + console.log("[rtc] low volume"); + void message.warn(t("low-volume")); + }); return () => void rtcAvatar.destroy(); }, [rtcAvatar, t]);