refactor(service-providers): mix screen audio into microphone

packages/flat-i18n/locales/en.json

@@ -410,7 +410,8 @@
     "tip-window-body": "You are sharing the current screen",
     "tip-window-button": "End Sharing",
     "self": "Share Screen",
-    "with-audio": "Share Screen with Audio"
+    "with-audio": "Share Screen with Audio",
+    "please-turn-on-mic": "Turn on the microphone so that others can hear your voice"
   },
   "app-store": "AppStore(Beta)",
   "recently-used": "Recently Used",

packages/flat-i18n/locales/zh-CN.json

@@ -410,7 +410,8 @@
     "tip-window-body": "正在共享屏幕",
     "tip-window-button": "结束共享",
     "self": "屏幕分享",
-    "with-audio": "包含桌面音频"
+    "with-audio": "包含桌面音频",
+    "please-turn-on-mic": "开启麦克风对方才能听到声音"
   },
   "app-store": "应用中心（测试版）",
   "recently-used": "最近使用",

packages/flat-pages/src/components/ShareScreen/ShareScreenPicker/index.tsx

@@ -1,12 +1,13 @@
 import "./style.less";
 
-import React, { useCallback, useLayoutEffect, useState } from "react";
+import React, { useCallback, useEffect, useLayoutEffect, useState } from "react";
 import classNames from "classnames";
 import { useTranslate } from "@netless/flat-i18n";
 import { ClassroomStore } from "@netless/flat-stores";
-import { Button, Modal, Spin, Checkbox, Row, Col } from "antd";
+import { Button, Modal, Spin, Checkbox, Row, Col, Select } from "antd";
 import { observer } from "mobx-react-lite";
-
+import { IServiceVideoChatDevice } from "@netless/flat-services";
+import { useSafePromise } from "flat-components";
 import { ScreenList } from "./ScreenList";
 
 interface ShareScreenPickerProps {
@@ -50,6 +51,9 @@ const ShareScreenPickerModel = observer<ShareScreenPickerProps>(function ShareSc
                             >
                                 {t("share-screen.with-audio")}
                             </Checkbox>
+                            {classroomStore.shareScreenWithAudio && (
+                                <ShareScreenSelectSpeaker classroom={classroomStore} />
+                            )}
                         </Col>
                         <Col>
                             <Button key="cancel" className="footer-button" onClick={closeModal}>
@@ -95,6 +99,46 @@ export const ShareScreenPicker = observer<ShareScreenPickerProps>(function Share
     ) : null;
 });
 
+interface ShareScreenSelectSpeakerProps {
+    classroom: ClassroomStore;
+}
+
+const ShareScreenSelectSpeaker = observer<ShareScreenSelectSpeakerProps>(
+    function ShareScreenSelectSpeaker({ classroom }) {
+        const sp = useSafePromise();
+        const [deviceId, setDeviceId] = useState<string>("");
+        const [devices, setDevices] = useState<IServiceVideoChatDevice[]>([]);
+
+        useEffect(() => {
+            sp(classroom.rtc.getSpeakerDevices()).then(setDevices);
+        }, [classroom.rtc, sp]);
+
+        useEffect(() => {
+            if (devices.length > 0 && !devices.some(e => e.deviceId === deviceId)) {
+                setDeviceId(
+                    devices.find(e => e.label.toLowerCase().includes("soundflower"))?.deviceId ??
+                        devices[0].deviceId,
+                );
+            }
+        }, [deviceId, devices]);
+
+        useEffect(() => {
+            const device = devices.find(e => e.deviceId === deviceId);
+            classroom.setShareScreenAudioDevice(device ? device.label : "");
+        }, [classroom, deviceId, devices]);
+
+        return (
+            <Select style={{ width: 180 }} value={deviceId} onChange={setDeviceId}>
+                {devices.map(({ deviceId, label }) => (
+                    <Select.Option key={deviceId} value={deviceId}>
+                        {label}
+                    </Select.Option>
+                ))}
+            </Select>
+        );
+    },
+);
+
 interface ConfirmButtonProps {
     isSelected: boolean;
     handleOk: () => void;

packages/flat-pages/src/components/ShareScreen/ShareScreenPicker/style.less

@@ -12,6 +12,11 @@
         min-width: 86px;
         height: 32px;
     }
+
+    .ant-col {
+        display: inline-flex;
+        align-items: center;
+    }
 }
 
 .share-screen-picker {

packages/flat-services/src/services/video-chat/share-screen.ts

@@ -21,7 +21,7 @@ export abstract class IServiceShareScreen {
     public readonly events = new Remitter<IServiceShareScreenData>();
 
     public abstract setParams(params: IServiceShareScreenParams | null): void;
-    public abstract enable(enabled: boolean, withAudio?: boolean): void;
+    public abstract enable(enabled: boolean, speakerName?: string): void;
     public abstract setElement(element: HTMLElement | null): void;
 
     public getScreenInfo(): Promise<IServiceShareScreenInfo[]> {

packages/flat-stores/src/classroom-store/index.ts

@@ -104,6 +104,7 @@ export class ClassroomStore {
 
     public selectedScreenInfo: IServiceShareScreenInfo | null = null;
     public shareScreenWithAudio = false;
+    public shareScreenAudioDeviceName = "";
 
     public shareScreenPickerVisible = false;
 
@@ -977,8 +978,23 @@ export class ClassroomStore {
         this.shareScreenWithAudio = force;
     };
 
+    public setShareScreenAudioDevice = (deviceName: string): void => {
+        this.shareScreenAudioDeviceName = deviceName;
+    };
+
     public toggleShareScreen = (force = !this.isScreenSharing): void => {
-        this.rtc.shareScreen.enable(force, this.shareScreenWithAudio);
+        // Guide the current user to turn on microphone on screen sharing with audio.
+        if (
+            force &&
+            this.shareScreenWithAudio &&
+            this.shareScreenPickerVisible &&
+            this.users.currentUser &&
+            this.users.currentUser.mic === false
+        ) {
+            message.info(FlatI18n.t("share-screen.please-turn-on-mic"));
+        }
+        const deviceName = this.shareScreenWithAudio ? this.shareScreenAudioDeviceName : undefined;
+        this.rtc.shareScreen.enable(force, deviceName);
         this.toggleShareScreenPicker(false);
     };
 

service-providers/agora-rtc/agora-rtc-electron/src/rtc-local-avatar.ts

@@ -8,6 +8,10 @@ export interface RTCAvatarConfig {
     element?: HTMLElement | null;
 }
 
+export interface LocalAudioDelegator {
+    enableLocalAudio: (enabled: boolean) => void;
+}
+
 export class RTCLocalAvatar implements IServiceVideoChatAvatar {
     private readonly _rtc: AgoraRTCElectron;
     private readonly _sideEffect = new SideEffectManager();
@@ -33,6 +37,24 @@ export class RTCLocalAvatar implements IServiceVideoChatAvatar {
         return this._rtc.getVolumeLevel() || 0;
     }
 
+    private _localAudioDelegator: LocalAudioDelegator | null = null;
+    public delegateLocalAudio(delegator: LocalAudioDelegator): () => void {
+        this._localAudioDelegator = delegator;
+        this.enableLocalAudio(Boolean(this._el$.value && this._shouldMic$.value));
+        return () => {
+            this._localAudioDelegator = null;
+            this.enableLocalAudio(Boolean(this._el$.value && this._shouldMic$.value));
+        };
+    }
+
+    private enableLocalAudio(enabled: boolean): void {
+        if (this._localAudioDelegator) {
+            this._localAudioDelegator.enableLocalAudio(enabled);
+        } else {
+            this._rtc.rtcEngine.enableLocalAudio(enabled);
+        }
+    }
+
     public constructor(config: RTCAvatarConfig) {
         this._rtc = config.rtc;
         this._el$ = new Val(config.element);
@@ -45,10 +67,10 @@ export class RTCLocalAvatar implements IServiceVideoChatAvatar {
                             this._rtc.setRole(IServiceVideoChatRole.Host);
                         }
                         this._rtc.rtcEngine.setupLocalVideo(el);
-                        this._rtc.rtcEngine.enableLocalAudio(this._shouldMic$.value);
+                        this.enableLocalAudio(this._shouldMic$.value);
                         this._rtc.rtcEngine.enableLocalVideo(this._shouldCamera$.value);
                     } else {
-                        this._rtc.rtcEngine.enableLocalAudio(false);
+                        this.enableLocalAudio(false);
                         this._rtc.rtcEngine.enableLocalVideo(false);
                         this._rtc.setRole(IServiceVideoChatRole.Audience);
                     }
@@ -64,9 +86,9 @@ export class RTCLocalAvatar implements IServiceVideoChatAvatar {
                     try {
                         if (shouldMic) {
                             this._rtc.setRole(IServiceVideoChatRole.Host);
-                            this._rtc.rtcEngine.enableLocalAudio(true);
+                            this.enableLocalAudio(true);
                         } else {
-                            this._rtc.rtcEngine.enableLocalAudio(false);
+                            this.enableLocalAudio(false);
                             if (!this._shouldCamera$.value) {
                                 this._rtc.setRole(IServiceVideoChatRole.Audience);
                             }
@@ -100,7 +122,7 @@ export class RTCLocalAvatar implements IServiceVideoChatAvatar {
 
         this._sideEffect.addDisposer(() => {
             this._rtc.rtcEngine.enableLocalVideo(false);
-            this._rtc.rtcEngine.enableLocalAudio(false);
+            this.enableLocalAudio(false);
             this._el$.setValue(null);
         });
     }

service-providers/agora-rtc/agora-rtc-electron/src/rtc-share-screen.ts

@@ -9,10 +9,11 @@ import type {
     ScreenSymbol,
     WindowInfo,
 } from "agora-electron-sdk/types/Api/native_type";
+import type { AgoraRTCElectron } from "./agora-rtc-electron";
 
 import { SideEffectManager } from "side-effect-manager";
 import { combine, Val } from "value-enhancer";
-import type { AgoraRTCElectron } from "./agora-rtc-electron";
+import { RTCLocalAvatar } from "./rtc-local-avatar";
 
 const rect = { x: 0, y: 0, width: 0, height: 0 };
 
@@ -44,7 +45,7 @@ export class AgoraRTCElectronShareScreen extends IServiceShareScreen {
 
     private readonly _screenInfo$ = new Val<IServiceShareScreenInfo | null>(null);
 
-    private _withAudio = false;
+    private _speakerName?: string;
 
     public constructor(config: AgoraRTCElectronShareScreenAvatarConfig) {
         super();
@@ -162,13 +163,11 @@ export class AgoraRTCElectronShareScreen extends IServiceShareScreen {
         this._screenInfo$.setValue(info);
     }
 
-    public enable(enabled: boolean, withAudio?: boolean): void {
+    public enable(enabled: boolean, speakerName?: string): void {
         if (this._el$.value && this._active$.value) {
             throw new Error("There already exists remote screen track.");
         }
-        if (typeof withAudio === "boolean") {
-            this._withAudio = withAudio;
-        }
+        this._speakerName = speakerName;
         this._enabled$.setValue(enabled);
     }
 
@@ -200,35 +199,13 @@ export class AgoraRTCElectronShareScreen extends IServiceShareScreen {
 
         const { roomUUID, token, uid } = this._params$.value;
 
-        const withAudio = this._withAudio;
+        const speakerName = this._speakerName;
 
         this._pTogglingShareScreen = new Promise<void>(resolve => {
             this.client.once("videoSourceJoinedSuccess", () => {
-                if (withAudio) {
-                    // Internally it enables "local audio" (microphone) in the video source instance
-                    // "video source" means the *second* instance of AgoraRtcEngine.
-                    // There could only be 2 instances in electron sdk.
-                    //
-                    // After that, the loopback recording can be *mixed* into the microphone stream.
-                    // After that, we adjust the microphone stream volume to 0 to make it only includes the loopback sound.
-                    this.client.videoSourceEnableAudio();
-
-                    // Install the virtual sound card "soundflower" on macOS.
-                    // https://api-ref.agora.io/en/voice-sdk/electron/3.x/classes/agorartcengine.html#videosourceenableloopbackrecording
-                    // https://docs.agora.io/cn/video-legacy/API%20Reference/electron/classes/agorartcengine.html#videosourceenableloopbackrecording
-                    let deviceName: string | null = null;
-                    if (this._rtc.isMac) {
-                        for (const device of this.client.getAudioPlaybackDevices()) {
-                            const name = (device as { devicename: string }).devicename;
-                            if (name.toLowerCase().includes("soundflower")) {
-                                deviceName = name;
-                                break;
-                            }
-                        }
-                    }
-                    this.client.videoSourceEnableLoopbackRecording(true, deviceName);
-                    // Because there's no way to disable microphone stream, adjust the volume to 0 to simulate.
-                    this.client.videoSourceAdjustRecordingSignalVolume(0);
+                if (speakerName) {
+                    this._delegateLocalAudio(true);
+                    this.client.enableLoopbackRecording(true, speakerName);
                 }
 
                 this.client.videoSourceSetVideoProfile(43, false);
@@ -251,11 +228,13 @@ export class AgoraRTCElectronShareScreen extends IServiceShareScreen {
             this.client.videoSourceInitialize(this._rtc.APP_ID);
             this.client.videoSourceSetChannelProfile(1);
             this.client.videoSourceJoin(token, roomUUID, "", Number(uid), {
-                publishLocalAudio: true,
+                publishLocalAudio: false,
                 publishLocalVideo: true,
                 autoSubscribeAudio: false,
                 autoSubscribeVideo: false,
             });
+            this.client.videoSourceMuteAllRemoteAudioStreams(true);
+            this.client.videoSourceMuteAllRemoteVideoStreams(true);
         });
         await this._pTogglingShareScreen;
         this._pTogglingShareScreen = undefined;
@@ -272,10 +251,10 @@ export class AgoraRTCElectronShareScreen extends IServiceShareScreen {
         this._lastEnabled = false;
 
         this._pTogglingShareScreen = new Promise<void>(resolve => {
-            this.client.videoSourceEnableLoopbackRecording(false);
+            this.client.enableLoopbackRecording(false);
+            this._delegateLocalAudio(false);
             this.client.stopScreenCapture2();
             this.client.once("videoSourceLeaveChannel", () => {
-                this.client.videoSourceDisableAudio();
                 this.client.videoSourceRelease();
                 resolve();
             });
@@ -284,6 +263,23 @@ export class AgoraRTCElectronShareScreen extends IServiceShareScreen {
         await this._pTogglingShareScreen;
         this._pTogglingShareScreen = undefined;
     }
+
+    private _stopDelegateLocalAudio: (() => void) | null = null;
+    private _delegateLocalAudio(enabled: boolean): void {
+        this._stopDelegateLocalAudio && this._stopDelegateLocalAudio();
+        const localAvatar = this._rtc.localAvatar as RTCLocalAvatar;
+        if (enabled) {
+            this._rtc.rtcEngine.enableLocalAudio(true);
+            this._stopDelegateLocalAudio = localAvatar.delegateLocalAudio({
+                enableLocalAudio: enabled => {
+                    this.client.adjustRecordingSignalVolume(enabled ? 100 : 0);
+                    this.client.adjustLoopbackRecordingSignalVolume(100);
+                },
+            });
+        } else {
+            this._stopDelegateLocalAudio = null;
+        }
+    }
 }
 
 type Truthy<T> = T extends false | "" | 0 | null | undefined ? never : T;