From 9011ae4e1f9a0f28f336adf0b11ec9b38c81d158 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 27 Aug 2025 14:01:01 +0200 Subject: [PATCH 01/45] temp Signed-off-by: Timo K --- src/livekit/MatrixAudioRenderer.tsx | 2 + src/livekit/livekitSubscriptionRoom.ts | 123 +++++++++++ src/livekit/openIDSFU.ts | 66 ++---- src/livekit/useLivekit.ts | 49 +++-- src/room/InCallView.tsx | 131 ++++-------- src/rtcSessionHelpers.ts | 119 +++++------ src/state/CallViewModel.ts | 283 ++++++++++++++++++------- yarn.lock | 2 +- 8 files changed, 461 insertions(+), 314 deletions(-) create mode 100644 src/livekit/livekitSubscriptionRoom.ts diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 24975509a..4b28a7339 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -78,6 +78,8 @@ export function MatrixAudioRenderer({ loggedInvalidIdentities.current.add(identity); }; + // TODO-MULTI-SFU this uses the livekit room form the context. We need to change it so it uses the + // livekit room explicitly so we can pass a list of rooms into the audio renderer and call useTracks for each room. const tracks = useTracks( [ Track.Source.Microphone, diff --git a/src/livekit/livekitSubscriptionRoom.ts b/src/livekit/livekitSubscriptionRoom.ts new file mode 100644 index 000000000..f92ff10e5 --- /dev/null +++ b/src/livekit/livekitSubscriptionRoom.ts @@ -0,0 +1,123 @@ +/* +Copyright 2023, 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + ConnectionState, + type E2EEManagerOptions, + ExternalE2EEKeyProvider, + LocalVideoTrack, + Room, + type RoomOptions, +} from "livekit-client"; +import { useEffect, useRef } from "react"; +import E2EEWorker from "livekit-client/e2ee-worker?worker"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; + +import { defaultLiveKitOptions } from "./options"; +import { type SFUConfig } from "./openIDSFU"; +import { type MuteStates } from "../room/MuteStates"; +import { useMediaDevices } from "../MediaDevicesContext"; +import { + type ECConnectionState, + useECConnectionState, +} from "./useECConnectionState"; +import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; +import { E2eeType } from "../e2ee/e2eeType"; +import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; +import { + useTrackProcessor, + useTrackProcessorSync, +} from "./TrackProcessorContext"; +import { observeTrackReference$ } from "../state/MediaViewModel"; +import { useUrlParams } from "../UrlParams"; +import { useInitial } from "../useInitial"; +import { getValue } from "../utils/observable"; +import { type SelectedDevice } from "../state/MediaDevices"; + +interface UseLivekitResult { + livekitPublicationRoom?: Room; + connState: ECConnectionState; +} + +// TODO-MULTI-SFU This is all the logic we need in the subscription connection logic (sync output devices) +// This is not used! (but summarizes what we need) +export function livekitSubscriptionRoom( + rtcSession: MatrixRTCSession, + muteStates: MuteStates, + sfuConfig: SFUConfig | undefined, + e2eeSystem: EncryptionSystem, +): UseLivekitResult { + // Only ever create the room once via useInitial. + // The call can end up with multiple livekit rooms. This is the particular room in + // which this participant publishes their media. + const publicationRoom = useInitial(() => { + logger.info("[LivekitRoom] Create LiveKit room"); + + let e2ee: E2EEManagerOptions | undefined; + if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { + logger.info("Created MatrixKeyProvider (per participant)"); + e2ee = { + keyProvider: new MatrixKeyProvider(), + worker: new E2EEWorker(), + }; + } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { + logger.info("Created ExternalE2EEKeyProvider (shared key)"); + e2ee = { + keyProvider: new ExternalE2EEKeyProvider(), + worker: new E2EEWorker(), + }; + } + + const roomOptions: RoomOptions = { + ...defaultLiveKitOptions, + audioOutput: { + // When using controlled audio devices, we don't want to set the + // deviceId here, because it will be set by the native app. + // (also the id does not need to match a browser device id) + deviceId: controlledAudioDevices + ? undefined + : getValue(devices.audioOutput.selected$)?.id, + }, + e2ee, + }; + // We have to create the room manually here due to a bug inside + // @livekit/components-react. JSON.stringify() is used in deps of a + // useEffect() with an argument that references itself, if E2EE is enabled + const room = new Room(roomOptions); + room.setE2EEEnabled(e2eeSystem.kind !== E2eeType.NONE).catch((e) => { + logger.error("Failed to set E2EE enabled on room", e); + }); + + return room; + }); + + // Setup and update the keyProvider which was create by `createRoom` + useEffect(() => { + const e2eeOptions = publicationRoom.options.e2ee; + if ( + e2eeSystem.kind === E2eeType.NONE || + !(e2eeOptions && "keyProvider" in e2eeOptions) + ) + return; + + if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { + (e2eeOptions.keyProvider as MatrixKeyProvider).setRTCSession(rtcSession); + } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { + (e2eeOptions.keyProvider as ExternalE2EEKeyProvider) + .setKey(e2eeSystem.secret) + .catch((e) => { + logger.error("Failed to set shared key for E2EE", e); + }); + } + }, [publicationRoom.options.e2ee, e2eeSystem, rtcSession]); + + return { + connState: connectionState, + livekitPublicationRoom: publicationRoom, + }; +} diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 2ebd6045e..a288ec57b 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -7,12 +7,7 @@ Please see LICENSE in the repository root for full details. import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { useEffect, useState } from "react"; -import { type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; -import { useActiveLivekitFocus } from "../room/useActiveFocus"; -import { useErrorBoundary } from "../useErrorBoundary"; import { FailToGetOpenIdToken } from "../utils/errors"; import { doNetworkOperationWithRetry } from "../utils/matrix"; @@ -34,38 +29,11 @@ export type OpenIDClientParts = Pick< "getOpenIdToken" | "getDeviceId" >; -export function useOpenIDSFU( - client: OpenIDClientParts, - rtcSession: MatrixRTCSession, -): SFUConfig | undefined { - const [sfuConfig, setSFUConfig] = useState(undefined); - - const activeFocus = useActiveLivekitFocus(rtcSession); - const { showErrorBoundary } = useErrorBoundary(); - - useEffect(() => { - if (activeFocus) { - getSFUConfigWithOpenID(client, activeFocus).then( - (sfuConfig) => { - setSFUConfig(sfuConfig); - }, - (e) => { - showErrorBoundary(new FailToGetOpenIdToken(e)); - logger.error("Failed to get SFU config", e); - }, - ); - } else { - setSFUConfig(undefined); - } - }, [client, activeFocus, showErrorBoundary]); - - return sfuConfig; -} - export async function getSFUConfigWithOpenID( client: OpenIDClientParts, - activeFocus: LivekitFocus, -): Promise { + serviceUrl: string, + livekitAlias: string, +): Promise { let openIdToken: IOpenIDToken; try { openIdToken = await doNetworkOperationWithRetry(async () => @@ -78,26 +46,16 @@ export async function getSFUConfigWithOpenID( } logger.debug("Got openID token", openIdToken); - try { - logger.info( - `Trying to get JWT from call's active focus URL of ${activeFocus.livekit_service_url}...`, - ); - const sfuConfig = await getLiveKitJWT( - client, - activeFocus.livekit_service_url, - activeFocus.livekit_alias, - openIdToken, - ); - logger.info(`Got JWT from call's active focus URL.`); + logger.info(`Trying to get JWT for focus ${serviceUrl}...`); + const sfuConfig = await getLiveKitJWT( + client, + serviceUrl, + livekitAlias, + openIdToken, + ); + logger.info(`Got JWT from call's active focus URL.`); - return sfuConfig; - } catch (e) { - logger.warn( - `Failed to get JWT from RTC session's active focus URL of ${activeFocus.livekit_service_url}.`, - e, - ); - return undefined; - } + return sfuConfig; } async function getLiveKitJWT( diff --git a/src/livekit/useLivekit.ts b/src/livekit/useLivekit.ts index 4c669b47d..0672a8eb8 100644 --- a/src/livekit/useLivekit.ts +++ b/src/livekit/useLivekit.ts @@ -50,11 +50,12 @@ import { getValue } from "../utils/observable"; import { type SelectedDevice } from "../state/MediaDevices"; interface UseLivekitResult { - livekitRoom?: Room; + livekitPublicationRoom?: Room; connState: ECConnectionState; } -export function useLivekit( +// TODO-MULTI-SFU This is not used anymore but the device syncing logic needs to be moved into the connection object. +export function useLivekitPublicationRoom( rtcSession: MatrixRTCSession, muteStates: MuteStates, sfuConfig: SFUConfig | undefined, @@ -83,7 +84,9 @@ export function useLivekit( const { processor } = useTrackProcessor(); // Only ever create the room once via useInitial. - const room = useInitial(() => { + // The call can end up with multiple livekit rooms. This is the particular room in + // which this participant publishes their media. + const publicationRoom = useInitial(() => { logger.info("[LivekitRoom] Create LiveKit room"); let e2ee: E2EEManagerOptions | undefined; @@ -135,7 +138,7 @@ export function useLivekit( // Setup and update the keyProvider which was create by `createRoom` useEffect(() => { - const e2eeOptions = room.options.e2ee; + const e2eeOptions = publicationRoom.options.e2ee; if ( e2eeSystem.kind === E2eeType.NONE || !(e2eeOptions && "keyProvider" in e2eeOptions) @@ -151,7 +154,7 @@ export function useLivekit( logger.error("Failed to set shared key for E2EE", e); }); } - }, [room.options.e2ee, e2eeSystem, rtcSession]); + }, [publicationRoom.options.e2ee, e2eeSystem, rtcSession]); // Sync the requested track processors with LiveKit useTrackProcessorSync( @@ -170,7 +173,7 @@ export function useLivekit( return track instanceof LocalVideoTrack ? track : null; }), ), - [room], + [publicationRoom], ), ), ); @@ -178,7 +181,7 @@ export function useLivekit( const connectionState = useECConnectionState( initialAudioInputId, initialMuteStates.audio.enabled, - room, + publicationRoom, sfuConfig, ); @@ -216,8 +219,11 @@ export function useLivekit( // It's important that we only do this in the connected state, because // LiveKit's internal mute states aren't consistent during connection setup, // and setting tracks to be enabled during this time causes errors. - if (room !== undefined && connectionState === ConnectionState.Connected) { - const participant = room.localParticipant; + if ( + publicationRoom !== undefined && + connectionState === ConnectionState.Connected + ) { + const participant = publicationRoom.localParticipant; // Always update the muteButtonState Ref so that we can read the current // state in awaited blocks. buttonEnabled.current = { @@ -275,7 +281,7 @@ export function useLivekit( audioMuteUpdating.current = true; trackPublication = await participant.setMicrophoneEnabled( buttonEnabled.current.audio, - room.options.audioCaptureDefaults, + publicationRoom.options.audioCaptureDefaults, ); audioMuteUpdating.current = false; break; @@ -283,7 +289,7 @@ export function useLivekit( videoMuteUpdating.current = true; trackPublication = await participant.setCameraEnabled( buttonEnabled.current.video, - room.options.videoCaptureDefaults, + publicationRoom.options.videoCaptureDefaults, ); videoMuteUpdating.current = false; break; @@ -347,11 +353,14 @@ export function useLivekit( logger.error("Failed to sync video mute state with LiveKit", e); }); } - }, [room, muteStates, connectionState]); + }, [publicationRoom, muteStates, connectionState]); useEffect(() => { // Sync the requested devices with LiveKit's devices - if (room !== undefined && connectionState === ConnectionState.Connected) { + if ( + publicationRoom !== undefined && + connectionState === ConnectionState.Connected + ) { const syncDevice = ( kind: MediaDeviceKind, selected$: Observable, @@ -359,15 +368,15 @@ export function useLivekit( selected$.subscribe((device) => { logger.info( "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", - room.getActiveDevice(kind), + publicationRoom.getActiveDevice(kind), " !== ", device?.id, ); if ( device !== undefined && - room.getActiveDevice(kind) !== device.id + publicationRoom.getActiveDevice(kind) !== device.id ) { - room + publicationRoom .switchActiveDevice(kind, device.id) .catch((e) => logger.error(`Failed to sync ${kind} device with LiveKit`, e), @@ -393,7 +402,7 @@ export function useLivekit( .pipe(switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER)) .subscribe(() => { const activeMicTrack = Array.from( - room.localParticipant.audioTrackPublications.values(), + publicationRoom.localParticipant.audioTrackPublications.values(), ).find((d) => d.source === Track.Source.Microphone)?.track; if ( @@ -408,7 +417,7 @@ export function useLivekit( // getUserMedia() call with deviceId: default to get the *new* default device. // Note that room.switchActiveDevice() won't work: Livekit will ignore it because // the deviceId hasn't changed (was & still is default). - room.localParticipant + publicationRoom.localParticipant .getTrackPublication(Track.Source.Microphone) ?.audioTrack?.restartTrack() .catch((e) => { @@ -422,10 +431,10 @@ export function useLivekit( for (const s of subscriptions) s?.unsubscribe(); }; } - }, [room, devices, connectionState, controlledAudioDevices]); + }, [publicationRoom, devices, connectionState, controlledAudioDevices]); return { connState: connectionState, - livekitRoom: room, + livekitPublicationRoom: publicationRoom, }; } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 6cdbb75c9..e12fc060e 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -5,9 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { RoomContext, useLocalParticipant } from "@livekit/components-react"; import { IconButton, Text, Tooltip } from "@vector-im/compound-web"; -import { ConnectionState, type Room as LivekitRoom } from "livekit-client"; import { type MatrixClient, type Room as MatrixRoom } from "matrix-js-sdk"; import { type FC, @@ -37,6 +35,7 @@ import { VolumeOnSolidIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import { useTranslation } from "react-i18next"; +import { ConnectionState } from "livekit-client"; import LogoMark from "../icons/LogoMark.svg?react"; import LogoType from "../icons/LogoType.svg?react"; @@ -59,14 +58,12 @@ import { type OTelGroupCallMembership } from "../otel/OTelGroupCallMembership"; import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal"; import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { RageshakeRequestModal } from "./RageshakeRequestModal"; -import { useLivekit } from "../livekit/useLivekit.ts"; import { useWakeLock } from "../useWakeLock"; import { useMergedRefs } from "../useMergedRefs"; import { type MuteStates } from "./MuteStates"; import { type MatrixInfo } from "./VideoPreview"; import { InviteButton } from "../button/InviteButton"; import { LayoutToggle } from "./LayoutToggle"; -import { useOpenIDSFU } from "../livekit/openIDSFU"; import { CallViewModel, type GridMode, @@ -108,9 +105,7 @@ import { useSetting, } from "../settings/settings"; import { ReactionsReader } from "../reactions/ReactionsReader"; -import { ConnectionLostError } from "../utils/errors.ts"; import { useTypedEventEmitter } from "../useEvents.ts"; -import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx"; import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships.ts"; import { useMediaDevices } from "../MediaDevicesContext.ts"; @@ -125,7 +120,7 @@ import { prefetchSounds } from "../soundUtils"; import { useAudioContext } from "../useAudioContext"; import ringtoneMp3 from "../sound/ringtone.mp3?url"; import ringtoneOgg from "../sound/ringtone.ogg?url"; -import { ObservableScope } from "../state/ObservableScope.ts"; +import { ConnectionLostError } from "../utils/errors.ts"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -138,92 +133,47 @@ export interface ActiveCallProps export const ActiveCall: FC = (props) => { const mediaDevices = useMediaDevices(); - const sfuConfig = useOpenIDSFU(props.client, props.rtcSession); - const { livekitRoom, connState } = useLivekit( - props.rtcSession, - props.muteStates, - sfuConfig, - props.e2eeSystem, - ); - const observableScope = useInitial(() => new ObservableScope()); - const connStateBehavior$ = useObservable( - (inputs$) => - observableScope.behavior( - inputs$.pipe(map(([connState]) => connState)), - connState, - ), - [connState], - ); const [vm, setVm] = useState(null); + const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = + useUrlParams(); + useEffect(() => { - logger.info( - `[Lifecycle] InCallView Component mounted, livekit room state ${livekitRoom?.state}`, + const reactionsReader = new ReactionsReader(props.rtcSession); + const vm = new CallViewModel( + props.rtcSession, + props.matrixRoom, + mediaDevices, + { + encryptionSystem: props.e2eeSystem, + autoLeaveWhenOthersLeft, + waitForCallPickup: waitForCallPickup && sendNotificationType === "ring", + }, + reactionsReader.raisedHands$, + reactionsReader.reactions$, + props.e2eeSystem, ); + setVm(vm); return (): void => { - logger.info( - `[Lifecycle] InCallView Component unmounted, livekit room state ${livekitRoom?.state}`, - ); - livekitRoom - ?.disconnect() - .then(() => { - logger.info( - `[Lifecycle] Disconnected from livekit room, state:${livekitRoom?.state}`, - ); - }) - .catch((e) => { - logger.error("[Lifecycle] Failed to disconnect from livekit room", e); - }); + vm.destroy(); + reactionsReader.destroy(); }; - }, [livekitRoom]); - - const { autoLeaveWhenOthersLeft, sendNotificationType, waitForCallPickup } = - useUrlParams(); - - useEffect(() => { - if (livekitRoom !== undefined) { - const reactionsReader = new ReactionsReader(props.rtcSession); - const vm = new CallViewModel( - props.rtcSession, - props.matrixRoom, - livekitRoom, - mediaDevices, - { - encryptionSystem: props.e2eeSystem, - autoLeaveWhenOthersLeft, - waitForCallPickup: - waitForCallPickup && sendNotificationType === "ring", - }, - connStateBehavior$, - reactionsReader.raisedHands$, - reactionsReader.reactions$, - ); - setVm(vm); - return (): void => { - vm.destroy(); - reactionsReader.destroy(); - }; - } }, [ props.rtcSession, props.matrixRoom, - livekitRoom, mediaDevices, props.e2eeSystem, - connStateBehavior$, autoLeaveWhenOthersLeft, sendNotificationType, waitForCallPickup, ]); - if (livekitRoom === undefined || vm === null) return null; + if (vm === null) return null; return ( - - - - - + + + ); }; @@ -233,7 +183,6 @@ export interface InCallViewProps { matrixInfo: MatrixInfo; rtcSession: MatrixRTCSession; matrixRoom: MatrixRoom; - livekitRoom: LivekitRoom; muteStates: MuteStates; /** Function to call when the user explicitly ends the call */ onLeave: (cause: "user", soundFile?: CallEventSounds) => void; @@ -248,7 +197,6 @@ export const InCallView: FC = ({ matrixInfo, rtcSession, matrixRoom, - livekitRoom, muteStates, onLeave, header: headerStyle, @@ -273,10 +221,6 @@ export const InCallView: FC = ({ const { hideScreensharing, showControls } = useUrlParams(); - const { isScreenShareEnabled, localParticipant } = useLocalParticipant({ - room: livekitRoom, - }); - const muteAllAudio = useBehavior(muteAllAudio$); // Call pickup state and display names are needed for waiting overlay/sounds const callPickupState = useBehavior(vm.callPickupState$); @@ -806,15 +750,16 @@ export const InCallView: FC = ({ ); const toggleScreensharing = useCallback(() => { - localParticipant - .setScreenShareEnabled(!isScreenShareEnabled, { - audio: true, - selfBrowserSurface: "include", - surfaceSwitching: "include", - systemAudio: "include", - }) - .catch(logger.error); - }, [localParticipant, isScreenShareEnabled]); + throw new Error("TODO-MULTI-SFU"); + // localParticipant + // .setScreenShareEnabled(!isScreenShareEnabled, { + // audio: true, + // selfBrowserSurface: "include", + // surfaceSwitching: "include", + // systemAudio: "include", + // }) + // .catch(logger.error); + }, []); const buttons: JSX.Element[] = []; @@ -841,7 +786,7 @@ export const InCallView: FC = ({ = ({ ) } - + {/* TODO-MULTI-SFU: */} {renderContent()} @@ -955,7 +900,7 @@ export const InCallView: FC = ({ onDismiss={closeSettings} tab={settingsTab} onTabChange={setSettingsTab} - livekitRoom={livekitRoom} + livekitRoom={undefined} // TODO-MULTI-SFU /> )} diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 73f58cea3..e5e567ef1 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details. import { isLivekitFocus, isLivekitFocusConfig, + LivekitFocusConfig, type LivekitFocus, type LivekitFocusActive, type MatrixRTCSession, @@ -31,24 +32,16 @@ export function makeActiveFocus(): LivekitFocusActive { }; } -async function makePreferredLivekitFoci( - rtcSession: MatrixRTCSession, - livekitAlias: string, -): Promise { - logger.log("Start building foci_preferred list: ", rtcSession.room.roomId); - - const preferredFoci: LivekitFocus[] = []; - - // Make the Focus from the running rtc session the highest priority one - // This minimizes how often we need to switch foci during a call. - const focusInUse = rtcSession.getFocusInUse(); - if (focusInUse && isLivekitFocus(focusInUse)) { - logger.log("Adding livekit focus from oldest member: ", focusInUse); - preferredFoci.push(focusInUse); - } +export function getLivekitAlias(rtcSession: MatrixRTCSession): string { + // For now we assume everything is a room-scoped call + return rtcSession.room.roomId; +} - // Warm up the first focus we owned, to ensure livekit room is created before any state event sent. - let toWarmUp: LivekitFocus | undefined; +async function makeFocusInternal( + rtcSession: MatrixRTCSession, +): Promise { + logger.log("Searching for a preferred focus"); + const livekitAlias = getLivekitAlias(rtcSession); // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); @@ -59,51 +52,42 @@ async function makePreferredLivekitFoci( FOCI_WK_KEY ]; if (Array.isArray(wellKnownFoci)) { - const validWellKnownFoci = wellKnownFoci - .filter((f) => !!f) - .filter(isLivekitFocusConfig) - .map((wellKnownFocus) => { - logger.log("Adding livekit focus from well known: ", wellKnownFocus); - return { ...wellKnownFocus, livekit_alias: livekitAlias }; - }); - if (validWellKnownFoci.length > 0) { - toWarmUp = validWellKnownFoci[0]; + const focus: LivekitFocusConfig | undefined = wellKnownFoci.find( + (f) => f && isLivekitFocusConfig(f), + ); + if (focus !== undefined) { + logger.log("Using LiveKit focus from .well-known: ", focus); + return { ...focus, livekit_alias: livekitAlias }; } - preferredFoci.push(...validWellKnownFoci); } } const urlFromConf = Config.get().livekit?.livekit_service_url; if (urlFromConf) { - const focusFormConf: LivekitFocus = { + const focusFromConf: LivekitFocus = { type: "livekit", livekit_service_url: urlFromConf, livekit_alias: livekitAlias, }; - toWarmUp = toWarmUp ?? focusFormConf; - logger.log("Adding livekit focus from config: ", focusFormConf); - preferredFoci.push(focusFormConf); + logger.log("Using LiveKit focus from config: ", focusFromConf); + return focusFromConf; } - if (toWarmUp) { - // this will call the jwt/sfu/get endpoint to pre create the livekit room. - await getSFUConfigWithOpenID(rtcSession.room.client, toWarmUp); - } - if (preferredFoci.length === 0) - throw new MatrixRTCFocusMissingError(domain ?? ""); - return Promise.resolve(preferredFoci); - - // TODO: we want to do something like this: - // - // const focusOtherMembers = await focusFromOtherMembers( - // rtcSession, - // livekitAlias, - // ); - // if (focusOtherMembers) preferredFoci.push(focusOtherMembers); + throw new MatrixRTCFocusMissingError(domain ?? ""); +} + +export async function makeFocus( + rtcSession: MatrixRTCSession, +): Promise { + const focus = await makeFocusInternal(rtcSession); + // this will call the jwt/sfu/get endpoint to pre create the livekit room. + await getSFUConfigWithOpenID(rtcSession.room.client, focus); + return focus; } export async function enterRTCSession( rtcSession: MatrixRTCSession, + focus: LivekitFocus, encryptMedia: boolean, useNewMembershipManager = true, useExperimentalToDeviceTransport = false, @@ -115,34 +99,27 @@ export async function enterRTCSession( // have started tracking by the time calls start getting created. // groupCallOTelMembership?.onJoinCall(); - // right now we assume everything is a room-scoped call - const livekitAlias = rtcSession.room.roomId; const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); const useDeviceSessionMemberEvents = features?.feature_use_device_session_member_events; - rtcSession.joinRoomSession( - await makePreferredLivekitFoci(rtcSession, livekitAlias), - makeActiveFocus(), - { - notificationType: getUrlParams().sendNotificationType, - useNewMembershipManager, - manageMediaKeys: encryptMedia, - ...(useDeviceSessionMemberEvents !== undefined && { - useLegacyMemberEvents: !useDeviceSessionMemberEvents, - }), - delayedLeaveEventRestartMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_ms, - delayedLeaveEventDelayMs: - matrixRtcSessionConfig?.delayed_leave_event_delay_ms, - delayedLeaveEventRestartLocalTimeoutMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, - networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, - makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, - membershipEventExpiryMs: - matrixRtcSessionConfig?.membership_event_expiry_ms, - useExperimentalToDeviceTransport, - }, - ); + rtcSession.joinRoomSession([focus], focus, { + notificationType: getUrlParams().sendNotificationType, + useNewMembershipManager, + manageMediaKeys: encryptMedia, + ...(useDeviceSessionMemberEvents !== undefined && { + useLegacyMemberEvents: !useDeviceSessionMemberEvents, + }), + delayedLeaveEventRestartMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_ms, + delayedLeaveEventDelayMs: + matrixRtcSessionConfig?.delayed_leave_event_delay_ms, + delayedLeaveEventRestartLocalTimeoutMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, + networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, + makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, + membershipEventExpiryMs: matrixRtcSessionConfig?.membership_event_expiry_ms, + useExperimentalToDeviceTransport, + }); if (widget) { try { await widget.api.transport.send(ElementWidgetActions.JoinCall, {}); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 462e4afc8..f0d2c0b7e 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -12,7 +12,9 @@ import { } from "@livekit/components-core"; import { ConnectionState, - type Room as LivekitRoom, + E2EEOptions, + ExternalE2EEKeyProvider, + Room as LivekitRoom, type LocalParticipant, ParticipantEvent, type RemoteParticipant, @@ -22,6 +24,7 @@ import { type EventTimelineSetHandlerMap, EventType, RoomEvent, + MatrixClient, RoomStateEvent, SyncState, type Room as MatrixRoom, @@ -63,6 +66,7 @@ import { import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, + isLivekitFocusConfig, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -116,7 +120,16 @@ import { observeSpeaker$ } from "./observeSpeaker"; import { shallowEquals } from "../utils/array"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; -import { constant, type Behavior } from "./Behavior"; +import { type Behavior } from "./Behavior"; +import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; +import { defaultLiveKitOptions } from "../livekit/options"; +import { + enterRTCSession, + getLivekitAlias, + makeFocus, +} from "../rtcSessionHelpers"; +import { E2eeType } from "../e2ee/e2eeType"; +import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -405,6 +418,31 @@ class ScreenShare { type MediaItem = UserMedia | ScreenShare; +function getE2eeOptions( + e2eeSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, +): E2EEOptions | undefined { + if (e2eeSystem.kind === E2eeType.NONE) return undefined; + + if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { + const keyProvider = new MatrixKeyProvider(); + keyProvider.setRTCSession(rtcSession); + return { + keyProvider, + worker: new E2EEWorker(), + }; + } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { + const keyProvider = new ExternalE2EEKeyProvider(); + keyProvider + .setKey(e2eeSystem.secret) + .catch((e) => logger.error("Failed to set shared key for E2EE", e)); + return { + keyProvider, + worker: new E2EEWorker(), + }; + } +} + function getRoomMemberFromRtcMember( rtcMember: CallMembership, room: MatrixRoom, @@ -427,8 +465,151 @@ function getRoomMemberFromRtcMember( return { id, member }; } -// TODO: Move wayyyy more business logic from the call and lobby views into here +class Connection { + // TODO-MULTI-SFU Add all device syncing logic from useLivekit + private readonly sfuConfig = getSFUConfigWithOpenID( + this.client, + this.serviceUrl, + this.livekitAlias, + ); + + public async startSubscribing(): Promise { + this.stopped = false; + const { url, jwt } = await this.sfuConfig; + if (!this.stopped) await this.livekitRoom.connect(url, jwt); + } + + public async startPublishing(): Promise { + this.stopped = false; + const { url, jwt } = await this.sfuConfig; + if (!this.stopped) + // TODO-MULTI-SFU this should not create a track? + await this.livekitRoom.localParticipant.createTracks({ + audio: { deviceId: "default" }, + }); + if (!this.stopped) await this.livekitRoom.connect(url, jwt); + } + + private stopped = false; + + public stop(): void { + void this.livekitRoom.disconnect(); + this.stopped = true; + } + + public readonly participants$ = connectedParticipantsObserver( + this.livekitRoom, + ).pipe(this.scope.state()); + + public constructor( + private readonly livekitRoom: LivekitRoom, + private readonly serviceUrl: string, + private readonly livekitAlias: string, + private readonly client: MatrixClient, + private readonly scope: ObservableScope, + ) {} +} + export class CallViewModel extends ViewModel { + private readonly e2eeOptions = getE2eeOptions( + this.encryptionSystem, + this.matrixRTCSession, + ); + + private readonly livekitAlias = getLivekitAlias(this.matrixRTCSession); + + private readonly livekitRoom = new LivekitRoom({ + ...defaultLiveKitOptions, + e2ee: this.e2eeOptions, + }); + + private readonly localFocus = makeFocus(this.matrixRTCSession); + + private readonly localConnection = this.localFocus.then( + (focus) => + new Connection( + this.livekitRoom, + focus.livekit_service_url, + this.livekitAlias, + this.matrixRTCSession.room.client, + this.scope, + ), + ); + + private readonly memberships$ = fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + ).pipe(map(() => this.matrixRTCSession.memberships)); + + private readonly foci$ = this.memberships$.pipe( + map( + (memberships) => + new Set( + memberships + .map((m) => this.matrixRTCSession.resolveActiveFocus(m)) + .filter((f) => f !== undefined && isLivekitFocusConfig(f)) + .map((f) => f.livekit_service_url), + ), + ), + ); + + private readonly remoteConnections$ = combineLatest([ + this.localFocus, + this.foci$, + ]).pipe( + accumulate(new Map(), (prev, [localFocus, foci]) => { + const stopped = new Map(prev); + const next = new Map(); + + for (const focus of foci) { + if (focus !== localFocus.livekit_service_url) { + stopped.delete(focus); + next.set( + focus, + prev.get(focus) ?? + new Connection( + new LivekitRoom({ + ...defaultLiveKitOptions, + e2ee: this.e2eeOptions, + }), + focus, + this.livekitAlias, + this.matrixRTCSession.room.client, + this.scope, + ), + ); + } + } + + for (const connection of stopped.values()) connection.stop(); + return next; + }), + ); + + private readonly joined$ = new Subject(); + + public join(): void { + this.joined$.next(); + } + + public leave(): void { + // TODO + } + + private readonly connectionInstructions$ = this.joined$.pipe( + switchMap(() => this.remoteConnections$), + startWith(new Map()), + pairwise(), + map(([prev, next]) => { + const start = new Set(next.values()); + for (const connection of prev.values()) start.delete(connection); + const stop = new Set(prev.values()); + for (const connection of next.values()) stop.delete(connection); + + return { start, stop }; + }), + ); + private readonly userId = this.matrixRoom.client.getUserId(); private readonly matrixConnected$ = this.scope.behavior( @@ -502,79 +683,13 @@ export class CallViewModel extends ViewModel { // in a split-brained state. private readonly pretendToBeDisconnected$ = this.reconnecting$; - /** - * The raw list of RemoteParticipants as reported by LiveKit - */ - private readonly rawRemoteParticipants$ = this.scope.behavior< - RemoteParticipant[] - >(connectedParticipantsObserver(this.livekitRoom), []); - - /** - * Lists of RemoteParticipants to "hold" on display, even if LiveKit claims that - * they've left - */ - private readonly remoteParticipantHolds$ = this.scope.behavior< - RemoteParticipant[][] - >( - this.livekitConnectionState$.pipe( - withLatestFrom(this.rawRemoteParticipants$), - mergeMap(([s, ps]) => { - // Whenever we switch focuses, we should retain all the previous - // participants for at least POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS ms to - // give their clients time to switch over and avoid jarring layout shifts - if (s === ECAddonConnectionState.ECSwitchingFocus) { - return concat( - // Hold these participants - of({ hold: ps }), - // Wait for time to pass and the connection state to have changed - forkJoin([ - timer(POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS), - this.livekitConnectionState$.pipe( - filter((s) => s !== ECAddonConnectionState.ECSwitchingFocus), - take(1), - ), - // Then unhold them - ]).pipe(map(() => ({ unhold: ps }))), - ); - } else { - return EMPTY; - } - }), - // Accumulate the hold instructions into a single list showing which - // participants are being held - accumulate([] as RemoteParticipant[][], (holds, instruction) => - "hold" in instruction - ? [instruction.hold, ...holds] - : holds.filter((h) => h !== instruction.unhold), - ), - ), - ); - /** * The RemoteParticipants including those that are being "held" on the screen */ private readonly remoteParticipants$ = this.scope - .behavior( - combineLatest( - [this.rawRemoteParticipants$, this.remoteParticipantHolds$], - (raw, holds) => { - const result = [...raw]; - const resultIds = new Set(result.map((p) => p.identity)); - - // Incorporate the held participants into the list - for (const hold of holds) { - for (const p of hold) { - if (!resultIds.has(p.identity)) { - result.push(p); - resultIds.add(p.identity); - } - } - } - - return result; - }, - ), - ) + .behavior< + RemoteParticipant[] + >(combineLatest([this.localConnection, this.remoteConnections$], (localConnection, remoteConnections) => combineLatest([localConnection.participants$, ...[...remoteConnections.values()].map((c) => c.participants$)], (...ps) => ps.flat(1))).pipe(switchAll(), startWith([]))) .pipe(pauseWhen(this.pretendToBeDisconnected$)); private readonly memberships$ = this.scope.behavior( @@ -1685,24 +1800,42 @@ export class CallViewModel extends ViewModel { ), filter((v) => v.playSounds), ); + // TODO-REBASE: expose connection state observable + public readonly livekitConnectionState$: Observable; public constructor( - // A call is permanently tied to a single Matrix room and LiveKit room + // A call is permanently tied to a single Matrix room private readonly matrixRTCSession: MatrixRTCSession, private readonly matrixRoom: MatrixRoom, - private readonly livekitRoom: LivekitRoom, private readonly mediaDevices: MediaDevices, private readonly options: CallViewModelOptions, - public readonly livekitConnectionState$: Behavior, private readonly handsRaisedSubject$: Observable< Record >, private readonly reactionsSubject$: Observable< Record >, + private readonly encryptionSystem: EncryptionSystem, ) { super(); + void this.localConnection.then((c) => c.startPublishing()); + this.connectionInstructions$ + .pipe(this.scope.bind()) + .subscribe(({ start, stop }) => { + for (const connection of start) connection.startSubscribing(); + for (const connection of stop) connection.stop(); + }); + combineLatest([this.localFocus, this.joined$]) + .pipe(this.scope.bind()) + .subscribe(([localFocus]) => { + enterRTCSession( + this.matrixRTCSession, + localFocus, + this.encryptionSystem.kind !== E2eeType.PER_PARTICIPANT, + ); + }); + // Pause upstream of all local media tracks when we're disconnected from // MatrixRTC, because it can be an unpleasant surprise for the app to say // 'reconnecting' and yet still be transmitting your media to others. diff --git a/yarn.lock b/yarn.lock index 5f224576c..668706b4b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10317,7 +10317,7 @@ __metadata: uuid: "npm:11" checksum: 10c0/ecd019c677c272c5598617dcde407dbe4b1b11460863b2a577e33f3fd8732c9d9073ec0221b471ec1eb24e2839eec20728db7f92c9348be83126547286e50805 languageName: node - linkType: hard + linkType: soft "matrix-widget-api@npm:^1.10.0, matrix-widget-api@npm:^1.13.0": version: 1.13.1 From 35319dd6b507bc6e0ec9552e285f31a95dc56d94 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 27 Aug 2025 14:29:22 +0200 Subject: [PATCH 02/45] Fix some errors in CallViewModel --- src/state/CallViewModel.ts | 39 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index f0d2c0b7e..11ca2428b 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -12,19 +12,20 @@ import { } from "@livekit/components-core"; import { ConnectionState, - E2EEOptions, + type E2EEOptions, ExternalE2EEKeyProvider, Room as LivekitRoom, type LocalParticipant, ParticipantEvent, type RemoteParticipant, } from "livekit-client"; +import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { ClientEvent, type EventTimelineSetHandlerMap, EventType, RoomEvent, - MatrixClient, + type MatrixClient, RoomStateEvent, SyncState, type Room as MatrixRoom, @@ -41,12 +42,10 @@ import { distinctUntilChanged, endWith, filter, - forkJoin, fromEvent, ignoreElements, map, merge, - mergeMap, of, pairwise, race, @@ -61,7 +60,6 @@ import { takeUntil, throttleTime, timer, - withLatestFrom, } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { @@ -75,10 +73,6 @@ import { } from "matrix-js-sdk/lib/matrixrtc"; import { ViewModel } from "./ViewModel"; -import { - ECAddonConnectionState, - type ECConnectionState, -} from "../livekit/useECConnectionState"; import { LocalUserMediaViewModel, type MediaViewModel, @@ -120,7 +114,7 @@ import { observeSpeaker$ } from "./observeSpeaker"; import { shallowEquals } from "../utils/array"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; -import { type Behavior } from "./Behavior"; +import { constant, type Behavior } from "./Behavior"; import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; import { defaultLiveKitOptions } from "../livekit/options"; import { @@ -130,6 +124,7 @@ import { } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; +import { ECConnectionState } from "../livekit/useECConnectionState"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -141,10 +136,6 @@ export interface CallViewModelOptions { waitForCallPickup?: boolean; } -// How long we wait after a focus switch before showing the real participant -// list again -const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; - // Do not play any sounds if the participant count has exceeded this // number. export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; @@ -497,9 +488,10 @@ class Connection { this.stopped = true; } - public readonly participants$ = connectedParticipantsObserver( - this.livekitRoom, - ).pipe(this.scope.state()); + public readonly participants$ = this.scope.behavior( + connectedParticipantsObserver(this.livekitRoom), + [], + ); public constructor( private readonly livekitRoom: LivekitRoom, @@ -649,9 +641,10 @@ export class CallViewModel extends ViewModel { private readonly connected$ = this.scope.behavior( and$( this.matrixConnected$, - this.livekitConnectionState$.pipe( - map((state) => state === ConnectionState.Connected), - ), + // TODO-MULTI-SFU + // this.livekitConnectionState$.pipe( + // map((state) => state === ConnectionState.Connected), + // ), ), ); @@ -1819,17 +1812,17 @@ export class CallViewModel extends ViewModel { ) { super(); - void this.localConnection.then((c) => c.startPublishing()); + void this.localConnection.then((c) => void c.startPublishing()); this.connectionInstructions$ .pipe(this.scope.bind()) .subscribe(({ start, stop }) => { - for (const connection of start) connection.startSubscribing(); + for (const connection of start) void connection.startSubscribing(); for (const connection of stop) connection.stop(); }); combineLatest([this.localFocus, this.joined$]) .pipe(this.scope.bind()) .subscribe(([localFocus]) => { - enterRTCSession( + void enterRTCSession( this.matrixRTCSession, localFocus, this.encryptionSystem.kind !== E2eeType.PER_PARTICIPANT, From d9b6302bf769b51c34d92d5f44759226dd2df085 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 27 Aug 2025 14:36:13 +0200 Subject: [PATCH 03/45] Fix crash? --- src/state/CallViewModel.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 11ca2428b..468bbd3f8 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -504,7 +504,7 @@ class Connection { export class CallViewModel extends ViewModel { private readonly e2eeOptions = getE2eeOptions( - this.encryptionSystem, + this.options.encryptionSystem, this.matrixRTCSession, ); @@ -1808,7 +1808,6 @@ export class CallViewModel extends ViewModel { private readonly reactionsSubject$: Observable< Record >, - private readonly encryptionSystem: EncryptionSystem, ) { super(); @@ -1825,7 +1824,7 @@ export class CallViewModel extends ViewModel { void enterRTCSession( this.matrixRTCSession, localFocus, - this.encryptionSystem.kind !== E2eeType.PER_PARTICIPANT, + this.options.encryptionSystem.kind !== E2eeType.PER_PARTICIPANT, ); }); From 376a4b4e4a538a226deea636accbe84176f994cb Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 27 Aug 2025 15:06:14 +0200 Subject: [PATCH 04/45] initial compiling version Signed-off-by: Timo K --- src/room/GroupCallView.tsx | 47 ++++++++++++++++++-------------------- src/room/RoomPage.tsx | 9 ++++---- 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index dbc3ea18c..8562d4f8a 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -41,7 +41,7 @@ import { ActiveCall } from "./InCallView"; import { MUTE_PARTICIPANT_COUNT, type MuteStates } from "./MuteStates"; import { useMediaDevices } from "../MediaDevicesContext"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; -import { enterRTCSession, leaveRTCSession } from "../rtcSessionHelpers"; +import { leaveRTCSession } from "../rtcSessionHelpers"; import { saveKeyForRoom, useRoomEncryptionSystem, @@ -90,7 +90,8 @@ interface Props { skipLobby: UrlParams["skipLobby"]; header: HeaderStyle; rtcSession: MatrixRTCSession; - isJoined: boolean; + joined: boolean; + setJoined: (value: boolean) => void; muteStates: MuteStates; widget: WidgetHelpers | null; } @@ -103,7 +104,8 @@ export const GroupCallView: FC = ({ skipLobby, header, rtcSession, - isJoined, + joined, + setJoined, muteStates, widget, }) => { @@ -213,12 +215,14 @@ export const GroupCallView: FC = ({ const enterRTCSessionOrError = useCallback( async (rtcSession: MatrixRTCSession): Promise => { try { - await enterRTCSession( - rtcSession, - perParticipantE2EE, - useNewMembershipManager, - useExperimentalToDeviceTransport, - ); + setJoined(true); + // TODO-MULTI-SFU what to do with error handling now that we don't use this function? + // await enterRTCSession( + // rtcSession, + // perParticipantE2EE, + // useNewMembershipManager, + // useExperimentalToDeviceTransport, + // ); } catch (e) { if (e instanceof ElementCallError) { setExternalError(e); @@ -230,12 +234,9 @@ export const GroupCallView: FC = ({ setExternalError(error); } } + return Promise.resolve(); }, - [ - perParticipantE2EE, - useExperimentalToDeviceTransport, - useNewMembershipManager, - ], + [setJoined], ); useEffect(() => { @@ -284,7 +285,7 @@ export const GroupCallView: FC = ({ const onJoin = (ev: CustomEvent): void => { (async (): Promise => { await defaultDeviceSetup(ev.detail.data as unknown as JoinCallData); - await enterRTCSessionOrError(rtcSession); + setJoined(true); widget.api.transport.reply(ev.detail, {}); })().catch((e) => { logger.error("Error joining RTC session on preload", e); @@ -296,11 +297,7 @@ export const GroupCallView: FC = ({ }; } else { // No lobby and no preload: we enter the rtc session right away - (async (): Promise => { - await enterRTCSessionOrError(rtcSession); - })().catch((e) => { - logger.error("Error joining RTC session immediately", e); - }); + setJoined(true); } } }, [ @@ -311,7 +308,7 @@ export const GroupCallView: FC = ({ perParticipantE2EE, mediaDevices, latestMuteStates, - enterRTCSessionOrError, + setJoined, useNewMembershipManager, ]); @@ -373,7 +370,7 @@ export const GroupCallView: FC = ({ ); useEffect(() => { - if (widget && isJoined) { + if (widget && joined) { // set widget to sticky once joined. widget.api.setAlwaysOnScreen(true).catch((e) => { logger.error("Error calling setAlwaysOnScreen(true)", e); @@ -391,7 +388,7 @@ export const GroupCallView: FC = ({ widget.lazyActions.off(ElementWidgetActions.HangupCall, onHangup); }; } - }, [widget, isJoined, rtcSession]); + }, [widget, joined, rtcSession]); const joinRule = useJoinRule(room); @@ -426,7 +423,7 @@ export const GroupCallView: FC = ({ client={client} matrixInfo={matrixInfo} muteStates={muteStates} - onEnter={() => void enterRTCSessionOrError(rtcSession)} + onEnter={() => setJoined(true)} confineToRoom={confineToRoom} hideHeader={header === HeaderStyle.None} participantCount={participantCount} @@ -444,7 +441,7 @@ export const GroupCallView: FC = ({ throw externalError; }; body = ; - } else if (isJoined) { + } else if (joined) { body = ( <> {shareModal} diff --git a/src/room/RoomPage.tsx b/src/room/RoomPage.tsx index b424c5116..480f8706c 100644 --- a/src/room/RoomPage.tsx +++ b/src/room/RoomPage.tsx @@ -61,10 +61,8 @@ export const RoomPage: FC = () => { const { avatarUrl, displayName: userDisplayName } = useProfile(client); const groupCallState = useLoadGroupCall(client, roomIdOrAlias, viaServers); - const isJoined = useMatrixRTCSessionJoinState( - groupCallState.kind === "loaded" ? groupCallState.rtcSession : undefined, - ); - const muteStates = useMuteStates(isJoined); + const [joined, setJoined] = useState(false); + const muteStates = useMuteStates(joined); useEffect(() => { // If we've finished loading, are not already authed and we've been given a display name as @@ -109,7 +107,8 @@ export const RoomPage: FC = () => { widget={widget} client={client!} rtcSession={groupCallState.rtcSession} - isJoined={isJoined} + joined={joined} + setJoined={setJoined} isPasswordlessUser={passwordlessUser} confineToRoom={confineToRoom} preload={preload} From c6e8c94fd6c59c7d29025302aef4618a34c5f6dd Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 27 Aug 2025 15:07:55 +0200 Subject: [PATCH 05/45] Fix makeFocus --- src/rtcSessionHelpers.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index e5e567ef1..c08fcd400 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -6,11 +6,10 @@ Please see LICENSE in the repository root for full details. */ import { - isLivekitFocus, isLivekitFocusConfig, - LivekitFocusConfig, + type LivekitFocusConfig, type LivekitFocus, - type LivekitFocusActive, + type LivekitFocusSelection, type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; @@ -25,7 +24,7 @@ import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; -export function makeActiveFocus(): LivekitFocusActive { +export function makeActiveFocus(): LivekitFocusSelection { return { type: "livekit", focus_selection: "oldest_membership", @@ -81,7 +80,11 @@ export async function makeFocus( ): Promise { const focus = await makeFocusInternal(rtcSession); // this will call the jwt/sfu/get endpoint to pre create the livekit room. - await getSFUConfigWithOpenID(rtcSession.room.client, focus); + await getSFUConfigWithOpenID( + rtcSession.room.client, + focus.livekit_service_url, + focus.livekit_alias, + ); return focus; } From cb91f1ad4ffe043d321f5ce9ef3a4895ce81483b Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 27 Aug 2025 15:33:41 +0200 Subject: [PATCH 06/45] Make it actually join the session --- src/state/CallViewModel.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 468bbd3f8..623e9e9e8 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -531,7 +531,10 @@ export class CallViewModel extends ViewModel { private readonly memberships$ = fromEvent( this.matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged, - ).pipe(map(() => this.matrixRTCSession.memberships)); + ).pipe( + startWith(null), + map(() => this.matrixRTCSession.memberships), + ); private readonly foci$ = this.memberships$.pipe( map( @@ -1873,5 +1876,8 @@ export class CallViewModel extends ViewModel { } } }); + + // Join automatically + this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? } } From 7b88420f6a255e4956e84481edaf609d178a0048 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 27 Aug 2025 16:56:57 +0200 Subject: [PATCH 07/45] first video! Signed-off-by: Timo K --- src/main.tsx | 6 +++--- src/room/InCallView.tsx | 2 -- src/state/CallViewModel.ts | 27 ++++++++++++++++++++++----- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/main.tsx b/src/main.tsx index 06275f599..e795a13cf 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -60,9 +60,9 @@ if (fatalError !== null) { Initializer.initBeforeReact() .then(() => { root.render( - - - , + // + , + // , ); }) .catch((e) => { diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index e12fc060e..1c8b41e96 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -107,7 +107,6 @@ import { import { ReactionsReader } from "../reactions/ReactionsReader"; import { useTypedEventEmitter } from "../useEvents.ts"; import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts"; -import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships.ts"; import { useMediaDevices } from "../MediaDevicesContext.ts"; import { EarpieceOverlay } from "./EarpieceOverlay.tsx"; import { useAppBarHidden, useAppBarSecondaryButton } from "../AppBar.tsx"; @@ -251,7 +250,6 @@ export const InCallView: FC = ({ useExperimentalToDeviceTransportSetting, ); const encryptionSystem = useRoomEncryptionSystem(matrixRoom.roomId); - const memberships = useMatrixRTCSessionMemberships(rtcSession); const showToDeviceEncryption = useMemo( () => diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 623e9e9e8..bade6ef9f 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -42,6 +42,7 @@ import { distinctUntilChanged, endWith, filter, + from, fromEvent, ignoreElements, map, @@ -367,6 +368,7 @@ class UserMedia { public destroy(): void { this.scope.end(); + this.vm.destroy(); } } @@ -473,12 +475,17 @@ class Connection { public async startPublishing(): Promise { this.stopped = false; const { url, jwt } = await this.sfuConfig; - if (!this.stopped) - // TODO-MULTI-SFU this should not create a track? - await this.livekitRoom.localParticipant.createTracks({ + if (!this.stopped) await this.livekitRoom.connect(url, jwt); + if (!this.stopped) { + const tracks = await this.livekitRoom.localParticipant.createTracks({ audio: { deviceId: "default" }, + video: true, }); - if (!this.stopped) await this.livekitRoom.connect(url, jwt); + for (const track of tracks) { + await this.livekitRoom.localParticipant.publishTrack(track); + } + // await this.livekitRoom.localParticipant.enableCameraAndMicrophone(); + } } private stopped = false; @@ -1814,7 +1821,17 @@ export class CallViewModel extends ViewModel { ) { super(); - void this.localConnection.then((c) => void c.startPublishing()); + void from(this.localConnection) + .pipe(this.scope.bind()) + .subscribe( + (c) => + void c + .startPublishing() + // eslint-disable-next-line no-console + .then(() => console.log("successfully started publishing")) + // eslint-disable-next-line no-console + .catch((e) => console.error("failed to start publishing", e)), + ); this.connectionInstructions$ .pipe(this.scope.bind()) .subscribe(({ start, stop }) => { From a10489d7df451f82bc3c282edc88786f7962fe44 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 27 Aug 2025 17:07:22 +0200 Subject: [PATCH 08/45] publish audio in remote rooms Signed-off-by: Timo K --- src/state/CallViewModel.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index bade6ef9f..095b7d03e 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -470,6 +470,10 @@ class Connection { this.stopped = false; const { url, jwt } = await this.sfuConfig; if (!this.stopped) await this.livekitRoom.connect(url, jwt); + const tracks = await this.livekitRoom.localParticipant.createTracks({ + audio: { deviceId: "default" }, + }); + await this.livekitRoom.localParticipant.publishTrack(tracks[0]); } public async startPublishing(): Promise { From 6bdfd7fbd8dbf0c2d0a50170e73767c7e00c805f Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 27 Aug 2025 17:08:08 +0200 Subject: [PATCH 09/45] add comment Signed-off-by: Timo K --- src/state/CallViewModel.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 095b7d03e..72351042d 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -470,6 +470,8 @@ class Connection { this.stopped = false; const { url, jwt } = await this.sfuConfig; if (!this.stopped) await this.livekitRoom.connect(url, jwt); + // TODO-MULTI-SFU in this livekit room we really do not want to publish any tracks. + // this is only for testing purposes const tracks = await this.livekitRoom.localParticipant.createTracks({ audio: { deviceId: "default" }, }); From 55b46b3f33ac48f9605d5bd9d44011bbeb398661 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 27 Aug 2025 18:41:03 +0200 Subject: [PATCH 10/45] introduce publishingParticipants$ Signed-off-by: Timo K --- src/state/CallViewModel.ts | 76 +++++++++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 14 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 72351042d..6c6301adb 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -66,6 +66,7 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, isLivekitFocusConfig, + LivekitFocusConfig, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -490,7 +491,6 @@ class Connection { for (const track of tracks) { await this.livekitRoom.localParticipant.publishTrack(track); } - // await this.livekitRoom.localParticipant.enableCameraAndMicrophone(); } } @@ -501,17 +501,45 @@ class Connection { this.stopped = true; } - public readonly participants$ = this.scope.behavior( + public readonly participantsIncludingJustSubscribers$ = this.scope.behavior( connectedParticipantsObserver(this.livekitRoom), [], ); + public readonly publishingParticipants$ = ( + memberships$: Behavior, + ): Observable => + this.scope.behavior( + combineLatest([ + connectedParticipantsObserver(this.livekitRoom), + memberships$, + ]).pipe( + map(([participants, memberships]) => { + const publishingMembers = membershipsFocusUrl( + memberships, + this.matrixRTCSession, + ) + .filter((f) => f.livekit_service_url === this.serviceUrl) + .map((f) => f.membership); + return publishingMembers + .map((m) => + participants.find( + (p) => p.identity === `${m.sender}:${m.deviceId}`, + ), + ) + .filter((p): p is RemoteParticipant => !!p); + }), + ), + [], + ); + public constructor( private readonly livekitRoom: LivekitRoom, private readonly serviceUrl: string, private readonly livekitAlias: string, private readonly client: MatrixClient, private readonly scope: ObservableScope, + private readonly matrixRTCSession: MatrixRTCSession, ) {} } @@ -523,7 +551,7 @@ export class CallViewModel extends ViewModel { private readonly livekitAlias = getLivekitAlias(this.matrixRTCSession); - private readonly livekitRoom = new LivekitRoom({ + private readonly localConnectionLivekitRoom = new LivekitRoom({ ...defaultLiveKitOptions, e2ee: this.e2eeOptions, }); @@ -533,11 +561,12 @@ export class CallViewModel extends ViewModel { private readonly localConnection = this.localFocus.then( (focus) => new Connection( - this.livekitRoom, + this.localConnectionLivekitRoom, focus.livekit_service_url, this.livekitAlias, this.matrixRTCSession.room.client, this.scope, + this.matrixRTCSession, ), ); @@ -553,10 +582,9 @@ export class CallViewModel extends ViewModel { map( (memberships) => new Set( - memberships - .map((m) => this.matrixRTCSession.resolveActiveFocus(m)) - .filter((f) => f !== undefined && isLivekitFocusConfig(f)) - .map((f) => f.livekit_service_url), + membershipsFocusUrl(memberships, this.matrixRTCSession).map( + (f) => f.livekit_service_url, + ), ), ), ); @@ -584,6 +612,7 @@ export class CallViewModel extends ViewModel { this.livekitAlias, this.matrixRTCSession.room.client, this.scope, + this.matrixRTCSession, ), ); } @@ -698,7 +727,7 @@ export class CallViewModel extends ViewModel { private readonly remoteParticipants$ = this.scope .behavior< RemoteParticipant[] - >(combineLatest([this.localConnection, this.remoteConnections$], (localConnection, remoteConnections) => combineLatest([localConnection.participants$, ...[...remoteConnections.values()].map((c) => c.participants$)], (...ps) => ps.flat(1))).pipe(switchAll(), startWith([]))) + >(combineLatest([this.localConnection, this.remoteConnections$], (localConnection, remoteConnections) => combineLatest([localConnection.participantsIncludingJustSubscribers$, ...[...remoteConnections.values()].map((c) => c.participantsIncludingJustSubscribers$)], (...ps) => ps.flat(1))).pipe(switchAll(), startWith([]))) .pipe(pauseWhen(this.pretendToBeDisconnected$)); private readonly memberships$ = this.scope.behavior( @@ -781,7 +810,7 @@ export class CallViewModel extends ViewModel { private readonly mediaItems$ = this.scope.behavior( combineLatest([ this.remoteParticipants$, - observeParticipantMedia(this.livekitRoom.localParticipant), + observeParticipantMedia(this.localConnectionLivekitRoom.localParticipant), duplicateTiles.value$, this.memberships$, showNonMemberTiles.value$, @@ -849,7 +878,7 @@ export class CallViewModel extends ViewModel { member, participant, this.options.encryptionSystem, - this.livekitRoom, + this.localConnectionLivekitRoom, this.mediaDevices, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( @@ -874,7 +903,7 @@ export class CallViewModel extends ViewModel { member, participant, this.options.encryptionSystem, - this.livekitRoom, + this.localConnectionLivekitRoom, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( map((m) => m.get(matrixIdentifier) ?? "[👻]"), @@ -916,7 +945,7 @@ export class CallViewModel extends ViewModel { undefined, participant, this.options.encryptionSystem, - this.livekitRoom, + this.localConnectionLivekitRoom, this.mediaDevices, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( @@ -1862,7 +1891,7 @@ export class CallViewModel extends ViewModel { // that our own media is displayed on screen. this.matrixConnected$.pipe(this.scope.bind()).subscribe((connected) => { const publications = - this.livekitRoom.localParticipant.trackPublications.values(); + this.localConnectionLivekitRoom.localParticipant.trackPublications.values(); if (connected) { for (const p of publications) { if (p.track?.isUpstreamPaused === true) { @@ -1904,3 +1933,22 @@ export class CallViewModel extends ViewModel { this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? } } + +const membershipsFocusUrl = ( + memberships: CallMembership[], + matrixRTCSession: MatrixRTCSession, +): { livekit_service_url: string; membership: CallMembership }[] => { + return memberships + .map( + (m) => + [matrixRTCSession.resolveActiveFocus(m), m] as [ + LivekitFocusConfig | undefined, + CallMembership, + ], + ) + .filter(([f, _]) => f !== undefined && isLivekitFocusConfig(f)) + .map(([f, m]) => ({ + livekit_service_url: f!.livekit_service_url, + membership: m, + })); +}; From 8ffb360114cf3d1c82e48b03344901995e3d4d8d Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 28 Aug 2025 10:34:43 +0200 Subject: [PATCH 11/45] add local storage + more readable + remoteParticipants + use publishingParticipants Signed-off-by: Timo K --- src/rtcSessionHelpers.ts | 10 ++++++++++ src/state/CallViewModel.ts | 39 +++++++++++++++++++++++++++----------- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index c08fcd400..07cc49fc2 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -44,6 +44,16 @@ async function makeFocusInternal( // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); + if (localStorage.getItem("timo-focus-url")) { + const timoFocusUrl = JSON.parse(localStorage.getItem("timo-focus-url")!); + const focusFromUrl: LivekitFocus = { + type: "livekit", + livekit_service_url: timoFocusUrl, + livekit_alias: livekitAlias, + }; + logger.log("Using LiveKit focus from localStorage: ", timoFocusUrl); + return focusFromUrl; + } if (domain) { // we use AutoDiscovery instead of relying on the MatrixClient having already // been fully configured and started diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 6c6301adb..62f008593 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -66,7 +66,7 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, isLivekitFocusConfig, - LivekitFocusConfig, + type LivekitFocusConfig, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -501,7 +501,7 @@ class Connection { this.stopped = true; } - public readonly participantsIncludingJustSubscribers$ = this.scope.behavior( + public readonly participantsIncludingSubscribers$ = this.scope.behavior( connectedParticipantsObserver(this.livekitRoom), [], ); @@ -570,12 +570,14 @@ export class CallViewModel extends ViewModel { ), ); - private readonly memberships$ = fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ).pipe( - startWith(null), - map(() => this.matrixRTCSession.memberships), + private readonly memberships$ = this.scope.behavior( + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + ).pipe( + startWith(null), + map(() => this.matrixRTCSession.memberships), + ), ); private readonly foci$ = this.memberships$.pipe( @@ -725,9 +727,24 @@ export class CallViewModel extends ViewModel { * The RemoteParticipants including those that are being "held" on the screen */ private readonly remoteParticipants$ = this.scope - .behavior< - RemoteParticipant[] - >(combineLatest([this.localConnection, this.remoteConnections$], (localConnection, remoteConnections) => combineLatest([localConnection.participantsIncludingJustSubscribers$, ...[...remoteConnections.values()].map((c) => c.participantsIncludingJustSubscribers$)], (...ps) => ps.flat(1))).pipe(switchAll(), startWith([]))) + .behavior( + combineLatest( + [this.localConnection, this.remoteConnections$], + (localConnection, remoteConnections) => { + const remoteConnectionsParticipants = [ + ...remoteConnections.values(), + ].map((c) => c.publishingParticipants$(this.memberships$)); + + return combineLatest( + [ + localConnection.publishingParticipants$(this.memberships$), + ...remoteConnectionsParticipants, + ], + (...ps) => ps.flat(1), + ); + }, + ).pipe(switchAll(), startWith([])), + ) .pipe(pauseWhen(this.pretendToBeDisconnected$)); private readonly memberships$ = this.scope.behavior( From 33bc78eec172a0ad5d502f1f93a59e9969f04df9 Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 28 Aug 2025 11:18:38 +0200 Subject: [PATCH 12/45] add logging Signed-off-by: Timo K --- src/rtcSessionHelpers.ts | 2 +- src/state/CallViewModel.ts | 35 ++++++++++++++++++++++++++++------- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 07cc49fc2..a88502afc 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -45,7 +45,7 @@ async function makeFocusInternal( // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); if (localStorage.getItem("timo-focus-url")) { - const timoFocusUrl = JSON.parse(localStorage.getItem("timo-focus-url")!); + const timoFocusUrl = localStorage.getItem("timo-focus-url")!; const focusFromUrl: LivekitFocus = { type: "livekit", livekit_service_url: timoFocusUrl, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 62f008593..7c04aafa0 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -459,8 +459,8 @@ function getRoomMemberFromRtcMember( return { id, member }; } +// TODO-MULTI-SFU Add all device syncing logic from useLivekit class Connection { - // TODO-MULTI-SFU Add all device syncing logic from useLivekit private readonly sfuConfig = getSFUConfigWithOpenID( this.client, this.serviceUrl, @@ -521,13 +521,34 @@ class Connection { ) .filter((f) => f.livekit_service_url === this.serviceUrl) .map((f) => f.membership); - return publishingMembers - .map((m) => - participants.find( - (p) => p.identity === `${m.sender}:${m.deviceId}`, - ), - ) + + const publishingP = publishingMembers + .map((m) => { + logger.log( + "Publishing participants: all participants at: ", + this.livekitAlias, + this.serviceUrl, + participants, + ); + return participants.find((p) => { + logger.log( + "Publishing participants: compare", + p.identity, + "===", + `${m.sender}:${m.deviceId}`, + ); + return p.identity === `${m.sender}:${m.deviceId}`; + }); + }) .filter((p): p is RemoteParticipant => !!p); + logger.log( + "Publishing participants: find participants for url ", + this.serviceUrl, + publishingMembers, + "Publishing participants: ", + publishingP, + ); + return publishingP; }), ), [], From a617a92e8884af823c76419b308d28e571114ec1 Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 28 Aug 2025 13:37:17 +0200 Subject: [PATCH 13/45] make it work Signed-off-by: Timo K --- src/state/CallViewModel.ts | 74 +++++++++++++++----------------------- 1 file changed, 28 insertions(+), 46 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 7c04aafa0..af588e62d 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -471,11 +471,6 @@ class Connection { this.stopped = false; const { url, jwt } = await this.sfuConfig; if (!this.stopped) await this.livekitRoom.connect(url, jwt); - // TODO-MULTI-SFU in this livekit room we really do not want to publish any tracks. - // this is only for testing purposes - const tracks = await this.livekitRoom.localParticipant.createTracks({ - audio: { deviceId: "default" }, - }); await this.livekitRoom.localParticipant.publishTrack(tracks[0]); } @@ -483,9 +478,10 @@ class Connection { this.stopped = false; const { url, jwt } = await this.sfuConfig; if (!this.stopped) await this.livekitRoom.connect(url, jwt); + if (!this.stopped) { const tracks = await this.livekitRoom.localParticipant.createTracks({ - audio: { deviceId: "default" }, + audio: true, video: true, }); for (const track of tracks) { @@ -524,30 +520,11 @@ class Connection { const publishingP = publishingMembers .map((m) => { - logger.log( - "Publishing participants: all participants at: ", - this.livekitAlias, - this.serviceUrl, - participants, - ); return participants.find((p) => { - logger.log( - "Publishing participants: compare", - p.identity, - "===", - `${m.sender}:${m.deviceId}`, - ); return p.identity === `${m.sender}:${m.deviceId}`; }); }) .filter((p): p is RemoteParticipant => !!p); - logger.log( - "Publishing participants: find participants for url ", - this.serviceUrl, - publishingMembers, - "Publishing participants: ", - publishingP, - ); return publishingP; }), ), @@ -612,21 +589,22 @@ export class CallViewModel extends ViewModel { ), ); - private readonly remoteConnections$ = combineLatest([ - this.localFocus, - this.foci$, - ]).pipe( - accumulate(new Map(), (prev, [localFocus, foci]) => { - const stopped = new Map(prev); - const next = new Map(); - - for (const focus of foci) { - if (focus !== localFocus.livekit_service_url) { - stopped.delete(focus); - next.set( - focus, - prev.get(focus) ?? - new Connection( + private readonly remoteConnections$ = this.scope.behavior( + combineLatest([this.localFocus, this.foci$]).pipe( + accumulate(new Map(), (prev, [localFocus, foci]) => { + const stopped = new Map(prev); + const next = new Map(); + for (const focus of foci) { + if (focus !== localFocus.livekit_service_url) { + stopped.delete(focus); + + let nextConnection = prev.get(focus); + if (!nextConnection) { + logger.log( + "SFU remoteConnections$ construct new connection: ", + focus, + ); + nextConnection = new Connection( new LivekitRoom({ ...defaultLiveKitOptions, e2ee: this.e2eeOptions, @@ -636,14 +614,18 @@ export class CallViewModel extends ViewModel { this.matrixRTCSession.room.client, this.scope, this.matrixRTCSession, - ), - ); + ); + } else { + logger.log("SFU remoteConnections$ use prev connection: ", focus); + } + next.set(focus, nextConnection); + } } - } - for (const connection of stopped.values()) connection.stop(); - return next; - }), + for (const connection of stopped.values()) connection.stop(); + return next; + }), + ), ); private readonly joined$ = new Subject(); From e4a54e3a195ee17b2ef3ddef6245f26b347352a1 Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 28 Aug 2025 13:52:12 +0200 Subject: [PATCH 14/45] refactor connnection class Signed-off-by: Timo K --- src/state/CallViewModel.ts | 96 ++---------------------------- src/state/Connection.ts | 116 +++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 90 deletions(-) create mode 100644 src/state/Connection.ts diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index af588e62d..034631415 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details. */ import { - connectedParticipantsObserver, observeParticipantEvents, observeParticipantMedia, } from "@livekit/components-core"; @@ -25,7 +24,6 @@ import { type EventTimelineSetHandlerMap, EventType, RoomEvent, - type MatrixClient, RoomStateEvent, SyncState, type Room as MatrixRoom, @@ -117,7 +115,6 @@ import { shallowEquals } from "../utils/array"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; import { constant, type Behavior } from "./Behavior"; -import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; import { defaultLiveKitOptions } from "../livekit/options"; import { enterRTCSession, @@ -126,7 +123,8 @@ import { } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { ECConnectionState } from "../livekit/useECConnectionState"; +import { type ECConnectionState } from "../livekit/useECConnectionState"; +import { Connection, PublishConnection } from "./Connection"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -459,88 +457,6 @@ function getRoomMemberFromRtcMember( return { id, member }; } -// TODO-MULTI-SFU Add all device syncing logic from useLivekit -class Connection { - private readonly sfuConfig = getSFUConfigWithOpenID( - this.client, - this.serviceUrl, - this.livekitAlias, - ); - - public async startSubscribing(): Promise { - this.stopped = false; - const { url, jwt } = await this.sfuConfig; - if (!this.stopped) await this.livekitRoom.connect(url, jwt); - await this.livekitRoom.localParticipant.publishTrack(tracks[0]); - } - - public async startPublishing(): Promise { - this.stopped = false; - const { url, jwt } = await this.sfuConfig; - if (!this.stopped) await this.livekitRoom.connect(url, jwt); - - if (!this.stopped) { - const tracks = await this.livekitRoom.localParticipant.createTracks({ - audio: true, - video: true, - }); - for (const track of tracks) { - await this.livekitRoom.localParticipant.publishTrack(track); - } - } - } - - private stopped = false; - - public stop(): void { - void this.livekitRoom.disconnect(); - this.stopped = true; - } - - public readonly participantsIncludingSubscribers$ = this.scope.behavior( - connectedParticipantsObserver(this.livekitRoom), - [], - ); - - public readonly publishingParticipants$ = ( - memberships$: Behavior, - ): Observable => - this.scope.behavior( - combineLatest([ - connectedParticipantsObserver(this.livekitRoom), - memberships$, - ]).pipe( - map(([participants, memberships]) => { - const publishingMembers = membershipsFocusUrl( - memberships, - this.matrixRTCSession, - ) - .filter((f) => f.livekit_service_url === this.serviceUrl) - .map((f) => f.membership); - - const publishingP = publishingMembers - .map((m) => { - return participants.find((p) => { - return p.identity === `${m.sender}:${m.deviceId}`; - }); - }) - .filter((p): p is RemoteParticipant => !!p); - return publishingP; - }), - ), - [], - ); - - public constructor( - private readonly livekitRoom: LivekitRoom, - private readonly serviceUrl: string, - private readonly livekitAlias: string, - private readonly client: MatrixClient, - private readonly scope: ObservableScope, - private readonly matrixRTCSession: MatrixRTCSession, - ) {} -} - export class CallViewModel extends ViewModel { private readonly e2eeOptions = getE2eeOptions( this.options.encryptionSystem, @@ -558,7 +474,7 @@ export class CallViewModel extends ViewModel { private readonly localConnection = this.localFocus.then( (focus) => - new Connection( + new PublishConnection( this.localConnectionLivekitRoom, focus.livekit_service_url, this.livekitAlias, @@ -1881,7 +1797,7 @@ export class CallViewModel extends ViewModel { .subscribe( (c) => void c - .startPublishing() + .start() // eslint-disable-next-line no-console .then(() => console.log("successfully started publishing")) // eslint-disable-next-line no-console @@ -1890,7 +1806,7 @@ export class CallViewModel extends ViewModel { this.connectionInstructions$ .pipe(this.scope.bind()) .subscribe(({ start, stop }) => { - for (const connection of start) void connection.startSubscribing(); + for (const connection of start) void connection.start(); for (const connection of stop) connection.stop(); }); combineLatest([this.localFocus, this.joined$]) @@ -1954,7 +1870,7 @@ export class CallViewModel extends ViewModel { } } -const membershipsFocusUrl = ( +export const membershipsFocusUrl = ( memberships: CallMembership[], matrixRTCSession: MatrixRTCSession, ): { livekit_service_url: string; membership: CallMembership }[] => { diff --git a/src/state/Connection.ts b/src/state/Connection.ts new file mode 100644 index 000000000..ff5ebb649 --- /dev/null +++ b/src/state/Connection.ts @@ -0,0 +1,116 @@ +// TODO-MULTI-SFU Add all device syncing logic from useLivekit +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { connectedParticipantsObserver } from "@livekit/components-core"; +import { + type Room as LivekitRoom, + type RemoteParticipant, +} from "livekit-client"; +import { type MatrixClient } from "matrix-js-sdk"; +import { + type CallMembership, + type MatrixRTCSession, +} from "matrix-js-sdk/lib/matrixrtc"; +import { combineLatest, map, type Observable } from "rxjs"; + +import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; +import { type Behavior } from "./Behavior"; +import { membershipsFocusUrl } from "./CallViewModel"; +import { type ObservableScope } from "./ObservableScope"; + +export class Connection { + protected readonly sfuConfig = getSFUConfigWithOpenID( + this.client, + this.serviceUrl, + this.livekitAlias, + ); + + public async start(): Promise { + this.stopped = false; + const { url, jwt } = await this.sfuConfig; + if (!this.stopped) await this.livekitRoom.connect(url, jwt); + } + + protected stopped = false; + + public stop(): void { + void this.livekitRoom.disconnect(); + this.stopped = true; + } + + public readonly participantsIncludingSubscribers$ = this.scope.behavior( + connectedParticipantsObserver(this.livekitRoom), + [], + ); + + public readonly publishingParticipants$ = ( + memberships$: Behavior, + ): Observable => + this.scope.behavior( + combineLatest([ + connectedParticipantsObserver(this.livekitRoom), + memberships$, + ]).pipe( + map(([participants, memberships]) => { + const publishingMembers = membershipsFocusUrl( + memberships, + this.matrixRTCSession, + ) + .filter((f) => f.livekit_service_url === this.serviceUrl) + .map((f) => f.membership); + + const publishingP = publishingMembers + .map((m) => { + return participants.find((p) => { + return p.identity === `${m.sender}:${m.deviceId}`; + }); + }) + .filter((p): p is RemoteParticipant => !!p); + return publishingP; + }), + ), + [], + ); + + public constructor( + protected readonly livekitRoom: LivekitRoom, + protected readonly serviceUrl: string, + protected readonly livekitAlias: string, + protected readonly client: MatrixClient, + protected readonly scope: ObservableScope, + protected readonly matrixRTCSession: MatrixRTCSession, + ) {} +} + +export class PublishConnection extends Connection { + public async start(): Promise { + this.stopped = false; + const { url, jwt } = await this.sfuConfig; + if (!this.stopped) await this.livekitRoom.connect(url, jwt); + + if (!this.stopped) { + const tracks = await this.livekitRoom.localParticipant.createTracks({ + audio: true, + video: true, + }); + for (const track of tracks) { + await this.livekitRoom.localParticipant.publishTrack(track); + } + } + } + + public stop(): void { + void this.livekitRoom.disconnect(); + this.stopped = true; + } + + public readonly participantsIncludingSubscribers$ = this.scope.behavior( + connectedParticipantsObserver(this.livekitRoom), + [], + ); +} From 802ebf828d5d6f19a00000675442d039df12ac14 Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 28 Aug 2025 15:32:46 +0200 Subject: [PATCH 15/45] refactor connection Signed-off-by: Timo K --- src/state/CallViewModel.ts | 103 +++++++++++++++++++++---------------- src/state/Connection.ts | 50 +++++++++--------- 2 files changed, 83 insertions(+), 70 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 034631415..2cbaf7384 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -63,6 +63,7 @@ import { import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, + isLivekitFocus, isLivekitFocusConfig, type LivekitFocusConfig, type MatrixRTCSession, @@ -476,11 +477,11 @@ export class CallViewModel extends ViewModel { (focus) => new PublishConnection( this.localConnectionLivekitRoom, - focus.livekit_service_url, + focus, this.livekitAlias, this.matrixRTCSession.room.client, this.scope, - this.matrixRTCSession, + this.membershipsAndFocusMap$, ), ); @@ -494,53 +495,67 @@ export class CallViewModel extends ViewModel { ), ); - private readonly foci$ = this.memberships$.pipe( - map( - (memberships) => - new Set( - membershipsFocusUrl(memberships, this.matrixRTCSession).map( - (f) => f.livekit_service_url, - ), - ), + private readonly membershipsAndFocusMap$ = this.scope.behavior( + this.memberships$.pipe( + map((memberships) => + memberships.flatMap((m) => { + const f = this.matrixRTCSession.resolveActiveFocus(m); + return f && isLivekitFocus(f) ? [{ membership: m, focus: f }] : []; + }), + ), ), ); + private readonly focusServiceUrls$ = this.membershipsAndFocusMap$.pipe( + map((v) => new Set(v.map(({ focus }) => focus.livekit_service_url))), + ); + private readonly remoteConnections$ = this.scope.behavior( - combineLatest([this.localFocus, this.foci$]).pipe( - accumulate(new Map(), (prev, [localFocus, foci]) => { - const stopped = new Map(prev); - const next = new Map(); - for (const focus of foci) { - if (focus !== localFocus.livekit_service_url) { - stopped.delete(focus); - - let nextConnection = prev.get(focus); - if (!nextConnection) { - logger.log( - "SFU remoteConnections$ construct new connection: ", - focus, - ); - nextConnection = new Connection( - new LivekitRoom({ - ...defaultLiveKitOptions, - e2ee: this.e2eeOptions, - }), - focus, - this.livekitAlias, - this.matrixRTCSession.room.client, - this.scope, - this.matrixRTCSession, - ); - } else { - logger.log("SFU remoteConnections$ use prev connection: ", focus); + combineLatest([this.localFocus, this.focusServiceUrls$]).pipe( + accumulate( + new Map(), + (prev, [localFocus, focusUrls]) => { + const stopped = new Map(prev); + const next = new Map(); + for (const focusUrl of focusUrls) { + if (focusUrl !== localFocus.livekit_service_url) { + stopped.delete(focusUrl); + + let nextConnection = prev.get(focusUrl); + if (!nextConnection) { + logger.log( + "SFU remoteConnections$ construct new connection: ", + focusUrl, + ); + nextConnection = new Connection( + new LivekitRoom({ + ...defaultLiveKitOptions, + e2ee: this.e2eeOptions, + }), + { + livekit_service_url: focusUrl, + livekit_alias: this.livekitAlias, + type: "livekit", + }, + this.livekitAlias, + this.matrixRTCSession.room.client, + this.scope, + this.membershipsAndFocusMap$, + ); + } else { + logger.log( + "SFU remoteConnections$ use prev connection: ", + focusUrl, + ); + } + next.set(focusUrl, nextConnection); } - next.set(focus, nextConnection); } - } - for (const connection of stopped.values()) connection.stop(); - return next; - }), + for (const connection of stopped.values()) connection.stop(); + return next; + }, + ), ), ); @@ -652,11 +667,11 @@ export class CallViewModel extends ViewModel { (localConnection, remoteConnections) => { const remoteConnectionsParticipants = [ ...remoteConnections.values(), - ].map((c) => c.publishingParticipants$(this.memberships$)); + ].map((c) => c.publishingParticipants$); return combineLatest( [ - localConnection.publishingParticipants$(this.memberships$), + localConnection.publishingParticipants$, ...remoteConnectionsParticipants, ], (...ps) => ps.flat(1), diff --git a/src/state/Connection.ts b/src/state/Connection.ts index ff5ebb649..6e114603e 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -13,20 +13,19 @@ import { } from "livekit-client"; import { type MatrixClient } from "matrix-js-sdk"; import { + type LivekitFocus, type CallMembership, - type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, map, type Observable } from "rxjs"; import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; import { type Behavior } from "./Behavior"; -import { membershipsFocusUrl } from "./CallViewModel"; import { type ObservableScope } from "./ObservableScope"; export class Connection { protected readonly sfuConfig = getSFUConfigWithOpenID( this.client, - this.serviceUrl, + this.focus.livekit_service_url, this.livekitAlias, ); @@ -48,42 +47,41 @@ export class Connection { [], ); - public readonly publishingParticipants$ = ( - memberships$: Behavior, - ): Observable => + public readonly publishingParticipants$: Observable = this.scope.behavior( combineLatest([ connectedParticipantsObserver(this.livekitRoom), - memberships$, + this.membershipsFocusMap$, ]).pipe( - map(([participants, memberships]) => { - const publishingMembers = membershipsFocusUrl( - memberships, - this.matrixRTCSession, - ) - .filter((f) => f.livekit_service_url === this.serviceUrl) - .map((f) => f.membership); - - const publishingP = publishingMembers - .map((m) => { - return participants.find((p) => { - return p.identity === `${m.sender}:${m.deviceId}`; - }); - }) - .filter((p): p is RemoteParticipant => !!p); - return publishingP; - }), + map(([participants, membershipsFocusMap]) => + membershipsFocusMap + // Find all members that claim to publish on this connection + .flatMap(({ membership, focus }) => + focus.livekit_service_url === this.focus.livekit_service_url + ? [membership] + : [], + ) + // Find all associated publishing livekit participant objects + .flatMap(({ sender, deviceId }) => { + const participant = participants.find( + (p) => p.identity === `${sender}:${deviceId}`, + ); + return participant ? [participant] : []; + }), + ), ), [], ); public constructor( protected readonly livekitRoom: LivekitRoom, - protected readonly serviceUrl: string, + protected readonly focus: LivekitFocus, protected readonly livekitAlias: string, protected readonly client: MatrixClient, protected readonly scope: ObservableScope, - protected readonly matrixRTCSession: MatrixRTCSession, + protected readonly membershipsFocusMap$: Behavior< + { membership: CallMembership; focus: LivekitFocus }[] + >, ) {} } From 598371b8077642967732a37d9b656403e9a5dceb Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 28 Aug 2025 17:45:14 +0200 Subject: [PATCH 16/45] lots of work. noone knows if it works. Signed-off-by: Timo K --- src/state/CallViewModel.ts | 387 +++++++++++++++++-------------------- src/state/Connection.ts | 364 +++++++++++++++++++++++++++++++--- 2 files changed, 515 insertions(+), 236 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 2cbaf7384..ad6943b9a 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -5,15 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - observeParticipantEvents, - observeParticipantMedia, -} from "@livekit/components-core"; +import { observeParticipantEvents } from "@livekit/components-core"; import { ConnectionState, type E2EEOptions, ExternalE2EEKeyProvider, - Room as LivekitRoom, + type Room as LivekitRoom, type LocalParticipant, ParticipantEvent, type RemoteParticipant, @@ -24,10 +21,10 @@ import { type EventTimelineSetHandlerMap, EventType, RoomEvent, + type RoomMember, RoomStateEvent, SyncState, type Room as MatrixRoom, - type RoomMember, } from "matrix-js-sdk"; import { BehaviorSubject, @@ -116,7 +113,7 @@ import { shallowEquals } from "../utils/array"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; import { constant, type Behavior } from "./Behavior"; -import { defaultLiveKitOptions } from "../livekit/options"; + import { enterRTCSession, getLivekitAlias, @@ -411,31 +408,6 @@ class ScreenShare { type MediaItem = UserMedia | ScreenShare; -function getE2eeOptions( - e2eeSystem: EncryptionSystem, - rtcSession: MatrixRTCSession, -): E2EEOptions | undefined { - if (e2eeSystem.kind === E2eeType.NONE) return undefined; - - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - const keyProvider = new MatrixKeyProvider(); - keyProvider.setRTCSession(rtcSession); - return { - keyProvider, - worker: new E2EEWorker(), - }; - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - const keyProvider = new ExternalE2EEKeyProvider(); - keyProvider - .setKey(e2eeSystem.secret) - .catch((e) => logger.error("Failed to set shared key for E2EE", e)); - return { - keyProvider, - worker: new E2EEWorker(), - }; - } -} - function getRoomMemberFromRtcMember( rtcMember: CallMembership, room: MatrixRoom, @@ -459,29 +431,25 @@ function getRoomMemberFromRtcMember( } export class CallViewModel extends ViewModel { - private readonly e2eeOptions = getE2eeOptions( + private readonly livekitAlias = getLivekitAlias(this.matrixRTCSession); + + private readonly livekitE2EERoomOptions = getE2eeOptions( this.options.encryptionSystem, this.matrixRTCSession, ); - private readonly livekitAlias = getLivekitAlias(this.matrixRTCSession); - - private readonly localConnectionLivekitRoom = new LivekitRoom({ - ...defaultLiveKitOptions, - e2ee: this.e2eeOptions, - }); - private readonly localFocus = makeFocus(this.matrixRTCSession); private readonly localConnection = this.localFocus.then( (focus) => new PublishConnection( - this.localConnectionLivekitRoom, focus, this.livekitAlias, this.matrixRTCSession.room.client, this.scope, this.membershipsAndFocusMap$, + this.mediaDevices, + this.livekitE2EERoomOptions, ), ); @@ -506,12 +474,12 @@ export class CallViewModel extends ViewModel { ), ); - private readonly focusServiceUrls$ = this.membershipsAndFocusMap$.pipe( + private readonly livekitServiceUrls$ = this.membershipsAndFocusMap$.pipe( map((v) => new Set(v.map(({ focus }) => focus.livekit_service_url))), ); private readonly remoteConnections$ = this.scope.behavior( - combineLatest([this.localFocus, this.focusServiceUrls$]).pipe( + combineLatest([this.localFocus, this.livekitServiceUrls$]).pipe( accumulate( new Map(), (prev, [localFocus, focusUrls]) => { @@ -528,10 +496,6 @@ export class CallViewModel extends ViewModel { focusUrl, ); nextConnection = new Connection( - new LivekitRoom({ - ...defaultLiveKitOptions, - e2ee: this.e2eeOptions, - }), { livekit_service_url: focusUrl, livekit_alias: this.livekitAlias, @@ -541,6 +505,7 @@ export class CallViewModel extends ViewModel { this.matrixRTCSession.room.client, this.scope, this.membershipsAndFocusMap$, + this.livekitE2EERoomOptions, ); } else { logger.log( @@ -657,29 +622,54 @@ export class CallViewModel extends ViewModel { // in a split-brained state. private readonly pretendToBeDisconnected$ = this.reconnecting$; - /** - * The RemoteParticipants including those that are being "held" on the screen - */ - private readonly remoteParticipants$ = this.scope - .behavior( - combineLatest( - [this.localConnection, this.remoteConnections$], - (localConnection, remoteConnections) => { - const remoteConnectionsParticipants = [ - ...remoteConnections.values(), - ].map((c) => c.publishingParticipants$); - - return combineLatest( - [ - localConnection.publishingParticipants$, - ...remoteConnectionsParticipants, - ], - (...ps) => ps.flat(1), + private readonly participants$ = this.scope + .behavior< + { + participant: LocalParticipant | RemoteParticipant; + member: RoomMember; + livekitRoom: LivekitRoom; + }[] + >( + from(this.localConnection).pipe( + switchMap((localConnection) => { + const memberError = (): never => { + throw new Error("No room member for call membership"); + }; + const localParticipant = { + participant: localConnection.livekitRoom.localParticipant, + member: + this.matrixRoom.getMember(this.userId ?? "") ?? memberError(), + livekitRoom: localConnection.livekitRoom, + }; + return this.remoteConnections$.pipe( + switchMap((connections) => + combineLatest( + [...connections.values()].map((c) => + c.publishingParticipants$.pipe( + map((ps) => + ps.map(({ participant, membership }) => ({ + participant, + member: + getRoomMemberFromRtcMember( + membership, + this.matrixRoom, + )?.member ?? memberError(), + livekitRoom: c.livekitRoom, + })), + ), + ), + ), + ), + ), + map((remoteParticipants) => [ + ...remoteParticipants.flat(1), + localParticipant, + ]), ); - }, - ).pipe(switchAll(), startWith([])), + }), + ), ) - .pipe(pauseWhen(this.pretendToBeDisconnected$)); + .pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)); private readonly memberships$ = this.scope.behavior( fromEvent( @@ -760,8 +750,7 @@ export class CallViewModel extends ViewModel { */ private readonly mediaItems$ = this.scope.behavior( combineLatest([ - this.remoteParticipants$, - observeParticipantMedia(this.localConnectionLivekitRoom.localParticipant), + this.participants$, duplicateTiles.value$, this.memberships$, showNonMemberTiles.value$, @@ -769,44 +758,17 @@ export class CallViewModel extends ViewModel { scan( ( prevItems, - [ - remoteParticipants, - { participant: localParticipant }, - duplicateTiles, - memberships, - showNonMemberTiles, - ], + [participants, duplicateTiles, memberships, showNonMemberTiles], ) => { - const newItems = new Map( + const newItems: Map = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { - const room = this.matrixRoom; - // m.rtc.members are the basis for calculating what is visible in the call - for (const rtcMember of memberships) { - const { member, id: livekitParticipantId } = - getRoomMemberFromRtcMember(rtcMember, room); - const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`; - - let participant: - | LocalParticipant - | RemoteParticipant - | undefined = undefined; - if (livekitParticipantId === "local") { - participant = localParticipant; - } else { - participant = remoteParticipants.find( - (p) => p.identity === livekitParticipantId, - ); - } - - if (!member) { - logger.error( - "Could not find member for media id: ", - livekitParticipantId, - ); - } + for (const { participant, member, livekitRoom } of participants) { + const matrixId = participant.isLocal + ? "local" + : participant.identity; for (let i = 0; i < 1 + duplicateTiles; i++) { - const indexedMediaId = `${livekitParticipantId}:${i}`; - let prevMedia = prevItems.get(indexedMediaId); + const mediaId = `${matrixId}:${i}`; + let prevMedia = prevItems.get(mediaId); if (prevMedia && prevMedia instanceof UserMedia) { prevMedia.updateParticipant(participant); if (prevMedia.vm.member === undefined) { @@ -819,33 +781,33 @@ export class CallViewModel extends ViewModel { } } yield [ - indexedMediaId, + mediaId, // We create UserMedia with or without a participant. // This will be the initial value of a BehaviourSubject. // Once a participant appears we will update the BehaviourSubject. (see above) prevMedia ?? new UserMedia( - indexedMediaId, + mediaId, member, participant, this.options.encryptionSystem, - this.localConnectionLivekitRoom, + livekitRoom, this.mediaDevices, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( - map((m) => m.get(matrixIdentifier) ?? "[👻]"), + map((m) => m.get(matrixId) ?? "[👻]"), ), this.handsRaised$.pipe( - map((v) => v[matrixIdentifier]?.time ?? null), + map((v) => v[matrixId]?.time ?? null), ), this.reactions$.pipe( - map((v) => v[matrixIdentifier] ?? undefined), + map((v) => v[matrixId] ?? undefined), ), ), ]; if (participant?.isScreenShareEnabled) { - const screenShareId = `${indexedMediaId}:screen-share`; + const screenShareId = `${mediaId}:screen-share`; yield [ screenShareId, prevItems.get(screenShareId) ?? @@ -854,10 +816,10 @@ export class CallViewModel extends ViewModel { member, participant, this.options.encryptionSystem, - this.localConnectionLivekitRoom, + livekitRoom, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( - map((m) => m.get(matrixIdentifier) ?? "[👻]"), + map((m) => m.get(matrixId) ?? "[👻]"), ), ), ]; @@ -879,47 +841,51 @@ export class CallViewModel extends ViewModel { // - If one wants to test scalability using the LiveKit CLI. // - If an experimental project does not yet do the MatrixRTC bits. // - If someone wants to debug if the LiveKit connection works but MatrixRTC room state failed to arrive. - const newNonMemberItems = showNonMemberTiles - ? new Map( - function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (const participant of remoteParticipants) { - for (let i = 0; i < 1 + duplicateTiles; i++) { - const maybeNonMemberParticipantId = - participant.identity + ":" + i; - if (!newItems.has(maybeNonMemberParticipantId)) { - const nonMemberId = maybeNonMemberParticipantId; - yield [ - nonMemberId, - prevItems.get(nonMemberId) ?? - new UserMedia( - nonMemberId, - undefined, - participant, - this.options.encryptionSystem, - this.localConnectionLivekitRoom, - this.mediaDevices, - this.pretendToBeDisconnected$, - this.memberDisplaynames$.pipe( - map( - (m) => m.get(participant.identity) ?? "[👻]", - ), - ), - of(null), - of(null), - ), - ]; - } - } - } - }.bind(this)(), - ) - : new Map(); - if (newNonMemberItems.size > 0) { - logger.debug("Added NonMember items: ", newNonMemberItems); - } + // TODO-MULTI-SFU + // const newNonMemberItems = showNonMemberTiles + // ? new Map( + // function* ( + // this: CallViewModel, + // ): Iterable<[string, MediaItem]> { + // for (const participant of remoteParticipants) { + // for (let i = 0; i < 1 + duplicateTiles; i++) { + // const maybeNonMemberParticipantId = + // participant.identity + ":" + i; + // if (!newItems.has(maybeNonMemberParticipantId)) { + // const nonMemberId = maybeNonMemberParticipantId; + // yield [ + // nonMemberId, + // prevItems.get(nonMemberId) ?? + // new UserMedia( + // nonMemberId, + // undefined, + // participant, + // this.options.encryptionSystem, + // localConnection.livekitRoom, + // this.mediaDevices, + // this.pretendToBeDisconnected$, + // this.memberDisplaynames$.pipe( + // map( + // (m) => + // m.get(participant.identity) ?? "[👻]", + // ), + // ), + // of(null), + // of(null), + // ), + // ]; + // } + // } + // } + // }.bind(this)(), + // ) + // : new Map(); + // if (newNonMemberItems.size > 0) { + // logger.debug("Added NonMember items: ", newNonMemberItems); + // } const combinedNew = new Map([ - ...newNonMemberItems.entries(), + // ...newNonMemberItems.entries(), ...newItems.entries(), ]); @@ -1840,66 +1806,77 @@ export class CallViewModel extends ViewModel { // We use matrixConnected$ rather than reconnecting$ because we want to // pause tracks during the initial joining sequence too until we're sure // that our own media is displayed on screen. - this.matrixConnected$.pipe(this.scope.bind()).subscribe((connected) => { - const publications = - this.localConnectionLivekitRoom.localParticipant.trackPublications.values(); - if (connected) { - for (const p of publications) { - if (p.track?.isUpstreamPaused === true) { - const kind = p.track.kind; - logger.log( - `Resumming ${kind} track (MatrixRTC connection present)`, - ); - p.track - .resumeUpstream() - .catch((e) => - logger.error( - `Failed to resume ${kind} track after MatrixRTC reconnection`, - e, - ), + void this.localConnection.then((localConnection) => + this.matrixConnected$.pipe(this.scope.bind()).subscribe((connected) => { + const publications = + localConnection.livekitRoom.localParticipant.trackPublications.values(); + if (connected) { + for (const p of publications) { + if (p.track?.isUpstreamPaused === true) { + const kind = p.track.kind; + logger.log( + `Resuming ${kind} track (MatrixRTC connection present)`, ); + p.track + .resumeUpstream() + .catch((e) => + logger.error( + `Failed to resume ${kind} track after MatrixRTC reconnection`, + e, + ), + ); + } } - } - } else { - for (const p of publications) { - if (p.track?.isUpstreamPaused === false) { - const kind = p.track.kind; - logger.log( - `Pausing ${kind} track (uncertain MatrixRTC connection)`, - ); - p.track - .pauseUpstream() - .catch((e) => - logger.error( - `Failed to pause ${kind} track after entering uncertain MatrixRTC connection`, - e, - ), + } else { + for (const p of publications) { + if (p.track?.isUpstreamPaused === false) { + const kind = p.track.kind; + logger.log( + `Pausing ${kind} track (uncertain MatrixRTC connection)`, ); + p.track + .pauseUpstream() + .catch((e) => + logger.error( + `Failed to pause ${kind} track after entering uncertain MatrixRTC connection`, + e, + ), + ); + } } } - } - }); + }), + ); // Join automatically this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? } } -export const membershipsFocusUrl = ( - memberships: CallMembership[], - matrixRTCSession: MatrixRTCSession, -): { livekit_service_url: string; membership: CallMembership }[] => { - return memberships - .map( - (m) => - [matrixRTCSession.resolveActiveFocus(m), m] as [ - LivekitFocusConfig | undefined, - CallMembership, - ], - ) - .filter(([f, _]) => f !== undefined && isLivekitFocusConfig(f)) - .map(([f, m]) => ({ - livekit_service_url: f!.livekit_service_url, - membership: m, - })); -}; +// TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes +// do we need this? + +function getE2eeOptions( + e2eeSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, +): E2EEOptions | undefined { + if (e2eeSystem.kind === E2eeType.NONE) return undefined; + + if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { + const keyProvider = new MatrixKeyProvider(); + keyProvider.setRTCSession(rtcSession); + return { + keyProvider, + worker: new E2EEWorker(), + }; + } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { + const keyProvider = new ExternalE2EEKeyProvider(); + keyProvider + .setKey(e2eeSystem.secret) + .catch((e) => logger.error("Failed to set shared key for E2EE", e)); + return { + keyProvider, + worker: new E2EEWorker(), + }; + } +} diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 6e114603e..700ee4efe 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -8,26 +8,42 @@ Please see LICENSE in the repository root for full details. import { connectedParticipantsObserver } from "@livekit/components-core"; import { - type Room as LivekitRoom, - type RemoteParticipant, + ConnectionState, + Room as LivekitRoom, + type RoomOptions, + type E2EEOptions, + RoomEvent, + Track, } from "livekit-client"; import { type MatrixClient } from "matrix-js-sdk"; import { type LivekitFocus, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, map, type Observable } from "rxjs"; +import { + BehaviorSubject, + combineLatest, + filter, + fromEvent, + map, + NEVER, + type Observable, + type Subscription, + switchMap, +} from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { type SelectedDevice, type MediaDevices } from "./MediaDevices"; import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; -import { type Behavior } from "./Behavior"; +import { constant, type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; +import { defaultLiveKitOptions } from "../livekit/options"; +import { getValue } from "../utils/observable"; +import { getUrlParams } from "../UrlParams"; +import { type MuteStates } from "../room/MuteStates"; export class Connection { - protected readonly sfuConfig = getSFUConfigWithOpenID( - this.client, - this.focus.livekit_service_url, - this.livekitAlias, - ); + protected stopped = false; public async start(): Promise { this.stopped = false; @@ -35,22 +51,44 @@ export class Connection { if (!this.stopped) await this.livekitRoom.connect(url, jwt); } - protected stopped = false; - public stop(): void { void this.livekitRoom.disconnect(); this.stopped = true; } - public readonly participantsIncludingSubscribers$ = this.scope.behavior( - connectedParticipantsObserver(this.livekitRoom), - [], + protected readonly sfuConfig = getSFUConfigWithOpenID( + this.client, + this.focus.livekit_service_url, + this.livekitAlias, ); - public readonly publishingParticipants$: Observable = - this.scope.behavior( + public readonly participantsIncludingSubscribers$; + public readonly publishingParticipants$; + public livekitRoom: LivekitRoom; + + public connectionState$: Behavior; + public constructor( + protected readonly focus: LivekitFocus, + protected readonly livekitAlias: string, + protected readonly client: MatrixClient, + protected readonly scope: ObservableScope, + protected readonly membershipsFocusMap$: Behavior< + { membership: CallMembership; focus: LivekitFocus }[] + >, + e2eeLivekitOptions: E2EEOptions | undefined, + ) { + this.livekitRoom = new LivekitRoom({ + ...defaultLiveKitOptions, + e2ee: e2eeLivekitOptions, + }); + this.participantsIncludingSubscribers$ = this.scope.behavior( + connectedParticipantsObserver(this.livekitRoom), + [], + ); + + this.publishingParticipants$ = this.scope.behavior( combineLatest([ - connectedParticipantsObserver(this.livekitRoom), + this.participantsIncludingSubscribers$, this.membershipsFocusMap$, ]).pipe( map(([participants, membershipsFocusMap]) => @@ -62,27 +100,24 @@ export class Connection { : [], ) // Find all associated publishing livekit participant objects - .flatMap(({ sender, deviceId }) => { + .flatMap((membership) => { const participant = participants.find( - (p) => p.identity === `${sender}:${deviceId}`, + (p) => + p.identity === `${membership.sender}:${membership.deviceId}`, ); - return participant ? [participant] : []; + return participant ? [{ participant, membership }] : []; }), ), ), [], ); - - public constructor( - protected readonly livekitRoom: LivekitRoom, - protected readonly focus: LivekitFocus, - protected readonly livekitAlias: string, - protected readonly client: MatrixClient, - protected readonly scope: ObservableScope, - protected readonly membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitFocus }[] - >, - ) {} + this.connectionState$ = this.scope.behavior( + fromEvent( + this.livekitRoom, + RoomEvent.ConnectionStateChanged, + ), + ); + } } export class PublishConnection extends Connection { @@ -111,4 +146,271 @@ export class PublishConnection extends Connection { connectedParticipantsObserver(this.livekitRoom), [], ); + private readonly muteStates$: Behavior; + private updatingMuteStates$ = new BehaviorSubject(false); + + public constructor( + protected readonly focus: LivekitFocus, + protected readonly livekitAlias: string, + protected readonly client: MatrixClient, + protected readonly scope: ObservableScope, + protected readonly membershipsFocusMap$: Behavior< + { membership: CallMembership; focus: LivekitFocus }[] + >, + protected readonly devices: MediaDevices, + e2eeLivekitOptions: E2EEOptions | undefined, + ) { + super( + focus, + livekitAlias, + client, + scope, + membershipsFocusMap$, + e2eeLivekitOptions, + ); + + // TODO-MULTI-SFU use actual mute states + this.muteStates$ = constant({ + audio: { enabled: true, setEnabled: (enabled) => {} }, + video: { enabled: true, setEnabled: (enabled) => {} }, + }); + + logger.info("[LivekitRoom] Create LiveKit room"); + const { controlledAudioDevices } = getUrlParams(); + + const roomOptions: RoomOptions = { + ...defaultLiveKitOptions, + videoCaptureDefaults: { + ...defaultLiveKitOptions.videoCaptureDefaults, + deviceId: getValue(this.devices.videoInput.selected$)?.id, + // TODO-MULTI-SFU add processor support back + // processor, + }, + audioCaptureDefaults: { + ...defaultLiveKitOptions.audioCaptureDefaults, + deviceId: getValue(devices.audioInput.selected$)?.id, + }, + audioOutput: { + // When using controlled audio devices, we don't want to set the + // deviceId here, because it will be set by the native app. + // (also the id does not need to match a browser device id) + deviceId: controlledAudioDevices + ? undefined + : getValue(devices.audioOutput.selected$)?.id, + }, + e2ee: e2eeLivekitOptions, + }; + // We have to create the room manually here due to a bug inside + // @livekit/components-react. JSON.stringify() is used in deps of a + // useEffect() with an argument that references itself, if E2EE is enabled + const room = new LivekitRoom(roomOptions); + room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => { + logger.error("Failed to set E2EE enabled on room", e); + }); + this.livekitRoom = room; + + // sync mute states TODO-MULTI_SFU This possibly can be simplified quite a bit. + combineLatest([ + this.connectionState$, + this.muteStates$, + this.updatingMuteStates$, + ]) + .pipe( + filter(([_c, _m, updating]) => !updating), + this.scope.bind(), + ) + .subscribe(([connectionState, muteStates, _]) => { + // Sync the requested mute states with LiveKit's mute states. We do it this + // way around rather than using LiveKit as the source of truth, so that the + // states can be consistent throughout the lobby and loading screens. + // It's important that we only do this in the connected state, because + // LiveKit's internal mute states aren't consistent during connection setup, + // and setting tracks to be enabled during this time causes errors. + if ( + this.livekitRoom !== undefined && + connectionState === ConnectionState.Connected + ) { + const participant = this.livekitRoom.localParticipant; + + enum MuteDevice { + Microphone, + Camera, + } + + const syncMuteState = async ( + iterCount: number, + type: MuteDevice, + ): Promise => { + // The approach for muting is to always bring the actual livekit state in sync with the button + // This allows for a very predictable and reactive behavior for the user. + // (the new state is the old state when pressing the button n times (where n is even)) + // (the new state is different to the old state when pressing the button n times (where n is uneven)) + // In case there are issues with the device there might be situations where setMicrophoneEnabled/setCameraEnabled + // return immediately. This should be caught with the Error("track with new mute state could not be published"). + // For now we are still using an iterCount to limit the recursion loop to 10. + // This could happen if the device just really does not want to turn on (hardware based issue) + // but the mute button is in unmute state. + // For now our fail mode is to just stay in this state. + // TODO: decide for a UX on how that fail mode should be treated (disable button, hide button, sync button back to muted without user input) + + if (iterCount > 10) { + logger.error( + "Stop trying to sync the input device with current mute state after 10 failed tries", + ); + return; + } + let devEnabled; + let btnEnabled; + switch (type) { + case MuteDevice.Microphone: + devEnabled = participant.isMicrophoneEnabled; + btnEnabled = muteStates.audio.enabled; + break; + case MuteDevice.Camera: + devEnabled = participant.isCameraEnabled; + btnEnabled = muteStates.video.enabled; + break; + } + if (devEnabled !== btnEnabled && !this.updatingMuteStates$.value) { + this.updatingMuteStates$.next(true); + + try { + let trackPublication; + switch (type) { + case MuteDevice.Microphone: + trackPublication = await participant.setMicrophoneEnabled( + btnEnabled, + this.livekitRoom.options.audioCaptureDefaults, + ); + break; + case MuteDevice.Camera: + trackPublication = await participant.setCameraEnabled( + btnEnabled, + this.livekitRoom.options.videoCaptureDefaults, + ); + break; + } + + if (trackPublication) { + // await participant.setMicrophoneEnabled can return immediately in some instances, + // so that participant.isMicrophoneEnabled !== buttonEnabled.current.audio still holds true. + // This happens if the device is still in a pending state + // "sleeping" here makes sure we let react do its thing so that participant.isMicrophoneEnabled is updated, + // so we do not end up in a recursion loop. + await new Promise((r) => setTimeout(r, 100)); + + // track got successfully changed to mute/unmute + // Run the check again after the change is done. Because the user + // can update the state (presses mute button) while the device is enabling + // itself we need might need to update the mute state right away. + // This async recursion makes sure that setCamera/MicrophoneEnabled is + // called as little times as possible. + await syncMuteState(iterCount + 1, type); + } else { + throw new Error( + "track with new mute state could not be published", + ); + } + } catch (e) { + if ((e as DOMException).name === "NotAllowedError") { + logger.error( + "Fatal error while syncing mute state: resetting", + e, + ); + if (type === MuteDevice.Microphone) { + muteStates.audio.setEnabled?.(false); + } else { + muteStates.video.setEnabled?.(false); + } + } else { + logger.error( + "Failed to sync audio mute state with LiveKit (will retry to sync in 1s):", + e, + ); + setTimeout(() => { + this.updatingMuteStates$.next(false); + }, 1000); + } + } + } + }; + + syncMuteState(0, MuteDevice.Microphone).catch((e) => { + logger.error("Failed to sync audio mute state with LiveKit", e); + }); + syncMuteState(0, MuteDevice.Camera).catch((e) => { + logger.error("Failed to sync video mute state with LiveKit", e); + }); + } + }); + + const syncDevice = ( + kind: MediaDeviceKind, + selected$: Observable, + ): Subscription => + selected$.pipe(this.scope.bind()).subscribe((device) => { + if (this.connectionState$.value !== ConnectionState.Connected) return; + logger.info( + "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", + this.livekitRoom.getActiveDevice(kind), + " !== ", + device?.id, + ); + if ( + device !== undefined && + this.livekitRoom.getActiveDevice(kind) !== device.id + ) { + this.livekitRoom + .switchActiveDevice(kind, device.id) + .catch((e) => + logger.error(`Failed to sync ${kind} device with LiveKit`, e), + ); + } + }); + + syncDevice("audioinput", devices.audioInput.selected$); + if (!controlledAudioDevices) + syncDevice("audiooutput", devices.audioOutput.selected$); + syncDevice("videoinput", devices.videoInput.selected$); + // Restart the audio input track whenever we detect that the active media + // device has changed to refer to a different hardware device. We do this + // for the sake of Chrome, which provides a "default" device that is meant + // to match the system's default audio input, whatever that may be. + // This is special-cased for only audio inputs because we need to dig around + // in the LocalParticipant object for the track object and there's not a nice + // way to do that generically. There is usually no OS-level default video capture + // device anyway, and audio outputs work differently. + devices.audioInput.selected$ + .pipe( + switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), + this.scope.bind(), + ) + .subscribe(() => { + if (this.connectionState$.value !== ConnectionState.Connected) return; + const activeMicTrack = Array.from( + this.livekitRoom.localParticipant.audioTrackPublications.values(), + ).find((d) => d.source === Track.Source.Microphone)?.track; + + if ( + activeMicTrack && + // only restart if the stream is still running: LiveKit will detect + // when a track stops & restart appropriately, so this is not our job. + // Plus, we need to avoid restarting again if the track is already in + // the process of being restarted. + activeMicTrack.mediaStreamTrack.readyState !== "ended" + ) { + // Restart the track, which will cause Livekit to do another + // getUserMedia() call with deviceId: default to get the *new* default device. + // Note that room.switchActiveDevice() won't work: Livekit will ignore it because + // the deviceId hasn't changed (was & still is default). + this.livekitRoom.localParticipant + .getTrackPublication(Track.Source.Microphone) + ?.audioTrack?.restartTrack() + .catch((e) => { + logger.error(`Failed to restart audio device track`, e); + }); + } + }); + } + // TODO-MULTI-SFU Sync the requested track processors with LiveKit } From 02f4c73759b686482bc13222b077f3cbfae1b9dc Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 28 Aug 2025 11:02:41 +0200 Subject: [PATCH 17/45] Add my own local storage SFU config stuff too --- src/rtcSessionHelpers.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index a88502afc..38d9e849e 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -42,6 +42,17 @@ async function makeFocusInternal( logger.log("Searching for a preferred focus"); const livekitAlias = getLivekitAlias(rtcSession); + const urlFromStorage = localStorage.getItem("robin-matrixrtc-auth"); + if (urlFromStorage !== null) { + const focusFromStorage: LivekitFocus = { + type: "livekit", + livekit_service_url: urlFromStorage, + livekit_alias: livekitAlias, + }; + logger.log("Using LiveKit focus from local storage: ", focusFromStorage); + return focusFromStorage; + } + // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); if (localStorage.getItem("timo-focus-url")) { From d46fe55a670190a58822312725de3d08ce12e0df Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 28 Aug 2025 17:40:35 +0200 Subject: [PATCH 18/45] Import unfinished mute states refactor --- src/room/GroupCallView.tsx | 8 +- src/room/RoomPage.tsx | 26 ++++-- src/state/MuteStates.ts | 163 +++++++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 7 deletions(-) create mode 100644 src/state/MuteStates.ts diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 8562d4f8a..40ec4627b 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -38,7 +38,7 @@ import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; import { useProfile } from "../profile/useProfile"; import { findDeviceByName } from "../utils/media"; import { ActiveCall } from "./InCallView"; -import { MUTE_PARTICIPANT_COUNT, type MuteStates } from "./MuteStates"; +import { type MuteStates } from "../state/MuteStates"; import { useMediaDevices } from "../MediaDevicesContext"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; import { leaveRTCSession } from "../rtcSessionHelpers"; @@ -76,6 +76,12 @@ import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts"; import { useAppBarTitle } from "../AppBar.tsx"; import { useBehavior } from "../useBehavior.ts"; +/** + * If there already are this many participants in the call, we automatically mute + * the user. + */ +export const MUTE_PARTICIPANT_COUNT = 8; + declare global { interface Window { rtcSession?: MatrixRTCSession; diff --git a/src/room/RoomPage.tsx b/src/room/RoomPage.tsx index 480f8706c..3924437bb 100644 --- a/src/room/RoomPage.tsx +++ b/src/room/RoomPage.tsx @@ -20,6 +20,8 @@ import { CheckIcon, UnknownSolidIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; +import { useObservable } from "observable-hooks"; +import { map } from "rxjs"; import { useClientLegacy } from "../ClientContext"; import { ErrorPage, FullScreenView, LoadingPage } from "../FullScreenView"; @@ -35,12 +37,13 @@ import { CallTerminatedMessage, useLoadGroupCall } from "./useLoadGroupCall"; import { LobbyView } from "./LobbyView"; import { E2eeType } from "../e2ee/e2eeType"; import { useProfile } from "../profile/useProfile"; -import { useMuteStates } from "./MuteStates"; import { useOptInAnalytics } from "../settings/settings"; import { Config } from "../config/Config"; import { Link } from "../button/Link"; import { ErrorView } from "../ErrorView"; -import { useMatrixRTCSessionJoinState } from "../useMatrixRTCSessionJoinState"; +import { useMediaDevices } from "../MediaDevicesContext"; +import { MuteStates } from "../state/MuteStates"; +import { ObservableScope } from "../state/ObservableScope"; export const RoomPage: FC = () => { const { confineToRoom, appPrompt, preload, header, displayName, skipLobby } = @@ -62,7 +65,18 @@ export const RoomPage: FC = () => { const groupCallState = useLoadGroupCall(client, roomIdOrAlias, viaServers); const [joined, setJoined] = useState(false); - const muteStates = useMuteStates(joined); + + const devices = useMediaDevices(); + const [muteStates, setMuteStates] = useState(null); + const joined$ = useObservable( + (inputs$) => inputs$.pipe(map(([joined]) => joined)), + [joined], + ); + useEffect(() => { + const scope = new ObservableScope(); + setMuteStates(new MuteStates(scope, devices, joined$)); + return (): void => scope.end(); + }, [devices, joined$]); useEffect(() => { // If we've finished loading, are not already authed and we've been given a display name as @@ -99,10 +113,10 @@ export const RoomPage: FC = () => { } }, [groupCallState.kind]); - const groupCallView = (): JSX.Element => { + const groupCallView = (): ReactNode => { switch (groupCallState.kind) { case "loaded": - return ( + return muteStates && ( { ); return ( - ; + set: ((enabled: boolean) => void) | null; + toggle: (() => void) | null; +} + +class MuteState { + private readonly enabledByDefault$ = + this.enabledByConfig && !getUrlParams().skipLobby + ? this.isJoined$.pipe(map((isJoined) => !isJoined)) + : of(false); + + private readonly data$: Observable = + this.device.available$.pipe( + map((available) => available.size > 0), + distinctUntilChanged(), + withLatestFrom( + this.enabledByDefault$, + (devicesConnected, enabledByDefault) => { + if (!devicesConnected) + return { enabled$: of(false), set: null, toggle: null }; + + const set$ = new Subject(); + const toggle$ = new Subject(); + return { + set: (enabled: boolean) => set$.next(enabled), + toggle: () => toggle$.next(), + // Assume the default value only once devices are actually connected + enabled$: merge( + set$, + toggle$.pipe(map(() => "toggle" as const)), + ).pipe( + accumulate(enabledByDefault, (prev, update) => + update === "toggle" ? !prev : update, + ), + ), + }; + }, + ), + this.scope.state(), + ); + + public readonly enabled$: Observable = this.data$.pipe( + switchMap(({ enabled$ }) => enabled$), + ); + + public readonly setEnabled$: Observable<((enabled: boolean) => void) | null> = + this.data$.pipe(map(({ set }) => set)); + + public readonly toggle$: Observable<(() => void) | null> = this.data$.pipe( + map(({ toggle }) => toggle), + ); + + public constructor( + private readonly scope: ObservableScope, + private readonly device: MediaDevice, + private readonly isJoined$: Observable, + private readonly enabledByConfig: boolean, + ) {} +} + +export class MuteStates { + public readonly audio = new MuteState( + this.scope, + this.mediaDevices.audioInput, + this.isJoined$, + Config.get().media_devices.enable_video, + ); + public readonly video = new MuteState( + this.scope, + this.mediaDevices.videoInput, + this.isJoined$, + Config.get().media_devices.enable_video, + ); + + public constructor( + private readonly scope: ObservableScope, + private readonly mediaDevices: MediaDevices, + private readonly isJoined$: Observable, + ) { + if (widget !== null) { + // Sync our mute states with the hosting client + const widgetApiState$ = combineLatest( + [this.audio.enabled$, this.video.enabled$], + (audio, video) => ({ audio_enabled: audio, video_enabled: video }), + ); + widgetApiState$.pipe(this.scope.bind()).subscribe((state) => { + widget!.api.transport + .send(ElementWidgetActions.DeviceMute, state) + .catch((e) => + logger.warn("Could not send DeviceMute action to widget", e), + ); + }); + + // Also sync the hosting client's mute states back with ours + const muteActions$ = fromEvent( + widget.lazyActions, + ElementWidgetActions.DeviceMute, + ) as Observable>; + muteActions$ + .pipe( + withLatestFrom( + widgetApiState$, + this.audio.setEnabled$, + this.video.setEnabled$, + ), + this.scope.bind(), + ) + .subscribe(([ev, state, setAudioEnabled, setVideoEnabled]) => { + // First copy the current state into our new state + const newState = { ...state }; + // Update new state if there are any requested changes from the widget + // action in `ev.detail.data`. + if ( + ev.detail.data.audio_enabled != null && + typeof ev.detail.data.audio_enabled === "boolean" && + setAudioEnabled !== null + ) { + newState.audio_enabled = ev.detail.data.audio_enabled; + setAudioEnabled(newState.audio_enabled); + } + if ( + ev.detail.data.video_enabled != null && + typeof ev.detail.data.video_enabled === "boolean" && + setVideoEnabled !== null + ) { + newState.video_enabled = ev.detail.data.video_enabled; + setVideoEnabled(newState.video_enabled); + } + widget!.api.transport.reply(ev.detail, newState); + }); + } + } +} From 386dc6c84d8db8ec751e283901f52837e6089f7b Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 28 Aug 2025 18:41:13 +0200 Subject: [PATCH 19/45] temp before holiday Signed-off-by: Timo K --- src/room/GroupCallView.tsx | 6 ++-- src/state/CallViewModel.ts | 67 +++++++++++++++++++------------------- src/state/MuteStates.ts | 37 +++++++++++---------- 3 files changed, 56 insertions(+), 54 deletions(-) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 40ec4627b..b2495f54f 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -132,7 +132,7 @@ export const GroupCallView: FC = ({ // This should use `useEffectEvent` (only available in experimental versions) useEffect(() => { if (memberships.length >= MUTE_PARTICIPANT_COUNT) - muteStates.audio.setEnabled?.(false); + muteStates.audio.setEnabled$.value?.(false); // eslint-disable-next-line react-hooks/exhaustive-deps }, []); @@ -261,7 +261,7 @@ export const GroupCallView: FC = ({ if (!deviceId) { logger.warn("Unknown audio input: " + audioInput); // override the default mute state - latestMuteStates.current!.audio.setEnabled?.(false); + latestMuteStates.current!.audio.setEnabled$.value?.(false); } else { logger.debug( `Found audio input ID ${deviceId} for name ${audioInput}`, @@ -275,7 +275,7 @@ export const GroupCallView: FC = ({ if (!deviceId) { logger.warn("Unknown video input: " + videoInput); // override the default mute state - latestMuteStates.current!.video.setEnabled?.(false); + latestMuteStates.current!.video.setEnabled$.value?.(false); } else { logger.debug( `Found video input ID ${deviceId} for name ${videoInput}`, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index ad6943b9a..b5112e5e6 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -18,13 +18,13 @@ import { import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { ClientEvent, - type EventTimelineSetHandlerMap, - EventType, - RoomEvent, type RoomMember, RoomStateEvent, SyncState, type Room as MatrixRoom, + type EventTimelineSetHandlerMap, + EventType, + RoomEvent, } from "matrix-js-sdk"; import { BehaviorSubject, @@ -61,8 +61,6 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, isLivekitFocus, - isLivekitFocusConfig, - type LivekitFocusConfig, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -112,11 +110,11 @@ import { observeSpeaker$ } from "./observeSpeaker"; import { shallowEquals } from "../utils/array"; import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; import { type MediaDevices } from "./MediaDevices"; -import { constant, type Behavior } from "./Behavior"; - +import { type Behavior } from "./Behavior"; import { enterRTCSession, getLivekitAlias, + leaveRTCSession, makeFocus, } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; @@ -453,16 +451,6 @@ export class CallViewModel extends ViewModel { ), ); - private readonly memberships$ = this.scope.behavior( - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ).pipe( - startWith(null), - map(() => this.matrixRTCSession.memberships), - ), - ); - private readonly membershipsAndFocusMap$ = this.scope.behavior( this.memberships$.pipe( map((memberships) => @@ -524,17 +512,19 @@ export class CallViewModel extends ViewModel { ), ); - private readonly joined$ = new Subject(); + private readonly join$ = new Subject(); public join(): void { - this.joined$.next(); + this.join$.next(); } + private readonly leave$ = new Subject(); + public leave(): void { - // TODO + this.leave$.next(); } - private readonly connectionInstructions$ = this.joined$.pipe( + private readonly connectionInstructions$ = this.join$.pipe( switchMap(() => this.remoteConnections$), startWith(new Map()), pairwise(), @@ -622,6 +612,17 @@ export class CallViewModel extends ViewModel { // in a split-brained state. private readonly pretendToBeDisconnected$ = this.reconnecting$; + private readonly memberships$ = this.scope.behavior( + fromEvent( + this.matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + ).pipe( + startWith(null), + pauseWhen(this.pretendToBeDisconnected$), + map(() => this.matrixRTCSession.memberships), + ), + ); + private readonly participants$ = this.scope .behavior< { @@ -671,17 +672,6 @@ export class CallViewModel extends ViewModel { ) .pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)); - private readonly memberships$ = this.scope.behavior( - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ).pipe( - startWith(null), - pauseWhen(this.pretendToBeDisconnected$), - map(() => this.matrixRTCSession.memberships), - ), - ); - /** * Displaynames for each member of the call. This will disambiguate * any displaynames that clashes with another member. Only members @@ -1790,7 +1780,7 @@ export class CallViewModel extends ViewModel { for (const connection of start) void connection.start(); for (const connection of stop) connection.stop(); }); - combineLatest([this.localFocus, this.joined$]) + combineLatest([this.localFocus, this.join$]) .pipe(this.scope.bind()) .subscribe(([localFocus]) => { void enterRTCSession( @@ -1799,6 +1789,17 @@ export class CallViewModel extends ViewModel { this.options.encryptionSystem.kind !== E2eeType.PER_PARTICIPANT, ); }); + this.join$.pipe(this.scope.bind()).subscribe(() => { + leaveRTCSession( + this.matrixRTCSession, + "user", // TODO-MULTI-SFU ? + // Wait for the sound in widget mode (it's not long) + Promise.resolve(), // TODO-MULTI-SFU + //Promise.all([audioPromise, posthogRequest]), + ).catch((e) => { + logger.error("Error leaving RTC session", e); + }); + }); // Pause upstream of all local media tracks when we're disconnected from // MatrixRTC, because it can be an unpleasant surprise for the app to say diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index d54251635..4ac095b0f 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -26,6 +26,7 @@ import { Config } from "../config/Config"; import { getUrlParams } from "../UrlParams"; import { type ObservableScope } from "./ObservableScope"; import { accumulate } from "../utils/observable"; +import { type Behavior } from "./Behavior"; interface MuteStateData { enabled$: Observable; @@ -33,13 +34,13 @@ interface MuteStateData { toggle: (() => void) | null; } -class MuteState { +class MuteState { private readonly enabledByDefault$ = this.enabledByConfig && !getUrlParams().skipLobby - ? this.isJoined$.pipe(map((isJoined) => !isJoined)) + ? this.joined$.pipe(map((isJoined) => !isJoined)) : of(false); - private readonly data$: Observable = + private readonly data$ = this.scope.behavior( this.device.available$.pipe( map((available) => available.size > 0), distinctUntilChanged(), @@ -52,8 +53,8 @@ class MuteState { const set$ = new Subject(); const toggle$ = new Subject(); return { - set: (enabled: boolean) => set$.next(enabled), - toggle: () => toggle$.next(), + set: (enabled: boolean): void => set$.next(enabled), + toggle: (): void => toggle$.next(), // Assume the default value only once devices are actually connected enabled$: merge( set$, @@ -66,24 +67,24 @@ class MuteState { }; }, ), - this.scope.state(), - ); + ), + ); - public readonly enabled$: Observable = this.data$.pipe( - switchMap(({ enabled$ }) => enabled$), + public readonly enabled$: Behavior = this.scope.behavior( + this.data$.pipe(switchMap(({ enabled$ }) => enabled$)), ); - public readonly setEnabled$: Observable<((enabled: boolean) => void) | null> = - this.data$.pipe(map(({ set }) => set)); + public readonly setEnabled$: Behavior<((enabled: boolean) => void) | null> = + this.scope.behavior(this.data$.pipe(map(({ set }) => set))); - public readonly toggle$: Observable<(() => void) | null> = this.data$.pipe( - map(({ toggle }) => toggle), + public readonly toggle$: Behavior<(() => void) | null> = this.scope.behavior( + this.data$.pipe(map(({ toggle }) => toggle)), ); public constructor( private readonly scope: ObservableScope, - private readonly device: MediaDevice, - private readonly isJoined$: Observable, + private readonly device: MediaDevice, + private readonly joined$: Observable, private readonly enabledByConfig: boolean, ) {} } @@ -92,20 +93,20 @@ export class MuteStates { public readonly audio = new MuteState( this.scope, this.mediaDevices.audioInput, - this.isJoined$, + this.joined$, Config.get().media_devices.enable_video, ); public readonly video = new MuteState( this.scope, this.mediaDevices.videoInput, - this.isJoined$, + this.joined$, Config.get().media_devices.enable_video, ); public constructor( private readonly scope: ObservableScope, private readonly mediaDevices: MediaDevices, - private readonly isJoined$: Observable, + private readonly joined$: Observable, ) { if (widget !== null) { // Sync our mute states with the hosting client From e08f16f889ad9d6c2e8b427ae9ff86e2c4222af3 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 29 Aug 2025 18:46:24 +0200 Subject: [PATCH 20/45] All my Friday work. Demoable! --- src/room/InCallView.tsx | 35 ++-- src/room/LobbyView.tsx | 48 +++--- src/room/VideoPreview.tsx | 11 +- src/state/CallViewModel.ts | 81 ++++++---- src/state/Connection.ts | 243 +++++++--------------------- src/state/MuteStates.ts | 65 ++++++-- src/state/ObservableScope.ts | 7 + src/useCallViewKeyboardShortcuts.ts | 28 ++-- 8 files changed, 220 insertions(+), 298 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 1c8b41e96..3d7ce8dba 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -60,7 +60,7 @@ import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { useWakeLock } from "../useWakeLock"; import { useMergedRefs } from "../useMergedRefs"; -import { type MuteStates } from "./MuteStates"; +import { type MuteStates } from "../state/MuteStates"; import { type MatrixInfo } from "./VideoPreview"; import { InviteButton } from "../button/InviteButton"; import { LayoutToggle } from "./LayoutToggle"; @@ -143,6 +143,7 @@ export const ActiveCall: FC = (props) => { props.rtcSession, props.matrixRoom, mediaDevices, + props.muteStates, { encryptionSystem: props.e2eeSystem, autoLeaveWhenOthersLeft, @@ -161,6 +162,7 @@ export const ActiveCall: FC = (props) => { props.rtcSession, props.matrixRoom, mediaDevices, + props.muteStates, props.e2eeSystem, autoLeaveWhenOthersLeft, sendNotificationType, @@ -265,22 +267,19 @@ export const InCallView: FC = ({ ], ); - const toggleMicrophone = useCallback( - () => muteStates.audio.setEnabled?.((e) => !e), - [muteStates], - ); - const toggleCamera = useCallback( - () => muteStates.video.setEnabled?.((e) => !e), - [muteStates], - ); + const audioEnabled = useBehavior(muteStates.audio.enabled$); + const videoEnabled = useBehavior(muteStates.video.enabled$); + const toggleAudio = useBehavior(muteStates.audio.toggle$); + const toggleVideo = useBehavior(muteStates.video.toggle$); + const setAudioEnabled = useBehavior(muteStates.audio.setEnabled$); // This function incorrectly assumes that there is a camera and microphone, which is not always the case. // TODO: Make sure that this module is resilient when it comes to camera/microphone availability! useCallViewKeyboardShortcuts( containerRef1, - toggleMicrophone, - toggleCamera, - (muted) => muteStates.audio.setEnabled?.(!muted), + toggleAudio, + toggleVideo, + setAudioEnabled, (reaction) => void sendReaction(reaction), () => void toggleRaisedHand(), ); @@ -764,18 +763,18 @@ export const InCallView: FC = ({ buttons.push( , , ); diff --git a/src/room/LobbyView.tsx b/src/room/LobbyView.tsx index 391cb391f..057c12063 100644 --- a/src/room/LobbyView.tsx +++ b/src/room/LobbyView.tsx @@ -31,7 +31,7 @@ import inCallStyles from "./InCallView.module.css"; import styles from "./LobbyView.module.css"; import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header"; import { type MatrixInfo, VideoPreview } from "./VideoPreview"; -import { type MuteStates } from "./MuteStates"; +import { type MuteStates } from "../state/MuteStates"; import { InviteButton } from "../button/InviteButton"; import { EndCallButton, @@ -50,8 +50,8 @@ import { useTrackProcessorSync, } from "../livekit/TrackProcessorContext"; import { usePageTitle } from "../usePageTitle"; -import { useLatest } from "../useLatest"; import { getValue } from "../utils/observable"; +import { useBehavior } from "../useBehavior"; interface Props { client: MatrixClient; @@ -88,14 +88,10 @@ export const LobbyView: FC = ({ const { t } = useTranslation(); usePageTitle(matrixInfo.roomName); - const onAudioPress = useCallback( - () => muteStates.audio.setEnabled?.((e) => !e), - [muteStates], - ); - const onVideoPress = useCallback( - () => muteStates.video.setEnabled?.((e) => !e), - [muteStates], - ); + const audioEnabled = useBehavior(muteStates.audio.enabled$); + const videoEnabled = useBehavior(muteStates.video.enabled$); + const toggleAudio = useBehavior(muteStates.audio.toggle$); + const toggleVideo = useBehavior(muteStates.video.toggle$); const [settingsModalOpen, setSettingsModalOpen] = useState(false); const [settingsTab, setSettingsTab] = useState(defaultSettingsTab); @@ -133,7 +129,7 @@ export const LobbyView: FC = ({ // re-open the devices when they change (see below). const initialAudioOptions = useInitial( () => - muteStates.audio.enabled && { + audioEnabled && { deviceId: getValue(devices.audioInput.selected$)?.id, }, ); @@ -150,27 +146,21 @@ export const LobbyView: FC = ({ // We also pass in a clone because livekit mutates the object passed in, // which would cause the devices to be re-opened on the next render. audio: Object.assign({}, initialAudioOptions), - video: muteStates.video.enabled && { + video: videoEnabled && { deviceId: videoInputId, processor: initialProcessor, }, }), - [ - initialAudioOptions, - muteStates.video.enabled, - videoInputId, - initialProcessor, - ], + [initialAudioOptions, videoEnabled, videoInputId, initialProcessor], ); - const latestMuteStates = useLatest(muteStates); const onError = useCallback( (error: Error) => { logger.error("Error while creating preview Tracks:", error); - latestMuteStates.current.audio.setEnabled?.(false); - latestMuteStates.current.video.setEnabled?.(false); + muteStates.audio.setEnabled$.value?.(false); + muteStates.video.setEnabled$.value?.(false); }, - [latestMuteStates], + [muteStates], ); const tracks = usePreviewTracks(localTrackOptions, onError); @@ -217,7 +207,7 @@ export const LobbyView: FC = ({
diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 9d3108649..ab083ffec 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -184,9 +184,6 @@ export const GroupCallView: FC = ({ } = useUrlParams(); const e2eeSystem = useRoomEncryptionSystem(room.roomId); const [useNewMembershipManager] = useSetting(useNewMembershipManagerSetting); - const [useExperimentalToDeviceTransport] = useSetting( - useExperimentalToDeviceTransportSetting, - ); // Save the password once we start the groupCallView useEffect(() => { @@ -223,12 +220,6 @@ export const GroupCallView: FC = ({ try { setJoined(true); // TODO-MULTI-SFU what to do with error handling now that we don't use this function? - // await enterRTCSession( - // rtcSession, - // perParticipantE2EE, - // useNewMembershipManager, - // useExperimentalToDeviceTransport, - // ); } catch (e) { if (e instanceof ElementCallError) { setExternalError(e); @@ -322,16 +313,16 @@ export const GroupCallView: FC = ({ const navigate = useNavigate(); - const onLeave = useCallback( - ( - cause: "user" | "error" = "user", - playSound: CallEventSounds = "left", - ): void => { + const onLeft = useCallback( + (reason: "timeout" | "user" | "allOthersLeft" | "decline"): void => { + let playSound: CallEventSounds = "left"; + if (reason === "timeout" || reason === "decline") playSound = reason; + + setLeft(true); const audioPromise = leaveSoundContext.current?.playSound(playSound); // In embedded/widget mode the iFrame will be killed right after the call ended prohibiting the posthog event from getting sent, // therefore we want the event to be sent instantly without getting queued/batched. const sendInstantly = !!widget; - setLeft(true); // we need to wait until the callEnded event is tracked on posthog. // Otherwise the iFrame gets killed before the callEnded event got tracked. const posthogRequest = new Promise((resolve) => { @@ -339,37 +330,33 @@ export const GroupCallView: FC = ({ room.roomId, rtcSession.memberships.length, sendInstantly, + rtcSession, ); window.setTimeout(resolve, 10); }); - // TODO-MULTI-SFU find a solution if this is supposed to happen here or in the view model. - leaveRTCSession( - rtcSession, - cause, - // Wait for the sound in widget mode (it's not long) - Promise.all([audioPromise, posthogRequest]), - ) - // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts. - .then(async () => { + void Promise.all([audioPromise, posthogRequest]) + .then(() => { if ( !isPasswordlessUser && !confineToRoom && !PosthogAnalytics.instance.isEnabled() ) { - await navigate("/"); + void navigate("/"); } }) - .catch((e) => { - logger.error("Error leaving RTC session", e); - }); + .catch(() => + logger.error( + "could failed to play leave audio or send posthog leave event", + ), + ); }, [ leaveSoundContext, widget, - rtcSession, room.roomId, + rtcSession, isPasswordlessUser, confineToRoom, navigate, @@ -457,7 +444,7 @@ export const GroupCallView: FC = ({ matrixInfo={matrixInfo} rtcSession={rtcSession as MatrixRTCSession} matrixRoom={room} - onLeave={onLeave} + onLeft={onLeft} header={header} muteStates={muteStates} e2eeSystem={e2eeSystem} @@ -518,7 +505,8 @@ export const GroupCallView: FC = ({ }} onError={ (/**error*/) => { - if (rtcSession.isJoined()) onLeave("error"); + // TODO this should not be "user". It needs a new case + if (rtcSession.isJoined()) onLeft("user"); } } > diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index 15a896a4a..6d2aaf0ab 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -177,7 +177,8 @@ function createInCallView(): RenderResult & { }} matrixRoom={room} livekitRoom={livekitRoom} - onLeave={function (): void { + participantCount={0} + onLeft={function (): void { throw new Error("Function not implemented."); }} onShareClick={null} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 1e92d110b..c12b44c9f 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -23,11 +23,7 @@ import useMeasure from "react-use-measure"; import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import classNames from "classnames"; import { BehaviorSubject, map } from "rxjs"; -import { - useObservable, - useObservableEagerState, - useSubscription, -} from "observable-hooks"; +import { useObservable, useObservableEagerState } from "observable-hooks"; import { logger } from "matrix-js-sdk/lib/logger"; import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport"; import { @@ -94,10 +90,7 @@ import { } from "../reactions/useReactionsSender"; import { ReactionsAudioRenderer } from "./ReactionAudioRenderer"; import { ReactionsOverlay } from "./ReactionsOverlay"; -import { - CallEventAudioRenderer, - type CallEventSounds, -} from "./CallEventAudioRenderer"; +import { CallEventAudioRenderer } from "./CallEventAudioRenderer"; import { debugTileLayout as debugTileLayoutSetting, useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting, @@ -129,6 +122,8 @@ const maxTapDurationMs = 400; export interface ActiveCallProps extends Omit { e2eeSystem: EncryptionSystem; + // TODO refactor those reasons into an enum + onLeft: (reason: "user" | "timeout" | "decline" | "allOthersLeft") => void; } export const ActiveCall: FC = (props) => { @@ -154,8 +149,11 @@ export const ActiveCall: FC = (props) => { reactionsReader.reactions$, ); setVm(vm); + + const sub = vm.left$.subscribe(props.onLeft); return (): void => { vm.destroy(); + sub.unsubscribe(); reactionsReader.destroy(); }; }, [ @@ -167,6 +165,7 @@ export const ActiveCall: FC = (props) => { autoLeaveWhenOthersLeft, sendNotificationType, waitForCallPickup, + props.onLeft, ]); if (vm === null) return null; @@ -185,8 +184,6 @@ export interface InCallViewProps { rtcSession: MatrixRTCSession; matrixRoom: MatrixRoom; muteStates: MuteStates; - /** Function to call when the user explicitly ends the call */ - onLeave: (cause: "user", soundFile?: CallEventSounds) => void; header: HeaderStyle; otelGroupCallMembership?: OTelGroupCallMembership; onShareClick: (() => void) | null; @@ -199,7 +196,7 @@ export const InCallView: FC = ({ rtcSession, matrixRoom, muteStates, - onLeave, + header: headerStyle, onShareClick, }) => { @@ -295,7 +292,6 @@ export const InCallView: FC = ({ const showFooter = useBehavior(vm.showFooter$); const earpieceMode = useBehavior(vm.earpieceMode$); const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$); - useSubscription(vm.autoLeave$, () => onLeave("user")); // We need to set the proper timings on the animation based upon the sound length. const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1; @@ -316,16 +312,6 @@ export const InCallView: FC = ({ }; }, [pickupPhaseAudio?.soundDuration, ringDuration]); - // When we enter timeout or decline we will leave the call. - useEffect((): void | (() => void) => { - if (callPickupState === "timeout") { - onLeave("user", "timeout"); - } - if (callPickupState === "decline") { - onLeave("user", "decline"); - } - }, [callPickupState, onLeave, pickupPhaseAudio]); - // When waiting for pickup, loop a waiting sound useEffect((): void | (() => void) => { if (callPickupState !== "ringing" || !pickupPhaseAudio) return; @@ -343,6 +329,7 @@ export const InCallView: FC = ({ if (callPickupState !== "ringing") return null; // Use room state for other participants data (the one that we likely want to reach) + // TODO: this screams it wants to be a behavior in the vm. const roomOthers = [ ...matrixRoom.getMembersWithMembership("join"), ...matrixRoom.getMembersWithMembership("invite"), @@ -816,7 +803,7 @@ export const InCallView: FC = ({ c.connectionState$), + startWith(ConnectionState.Disconnected), + ), + ); + // TODO-MULTI-SFU make sure that we consider the room memberships here as well (so that here we only have valid memberships) // this also makes it possible to use this memberships$ list in all observables based on it. // there should be no other call to: this.matrixRTCSession.memberships! @@ -541,12 +550,19 @@ export class CallViewModel extends ViewModel { this.join$.next(); } - private readonly leave$ = new Subject(); + private readonly leave$ = new Subject< + "decline" | "timeout" | "user" | "allOthersLeft" + >(); public leave(): void { - this.leave$.next(); + this.leave$.next("user"); } + private readonly _left$ = new Subject< + "decline" | "timeout" | "user" | "allOthersLeft" + >(); + public left$ = this._left$.asObservable(); + private readonly connectionInstructions$ = this.join$.pipe( switchMap(() => this.remoteConnections$), startWith(new Map()), @@ -628,10 +644,9 @@ export class CallViewModel extends ViewModel { private readonly connected$ = this.scope.behavior( and$( this.matrixConnected$, - // TODO-MULTI-SFU - // this.livekitConnectionState$.pipe( - // map((state) => state === ConnectionState.Connected), - // ), + this.livekitConnectionState$.pipe( + map((state) => state === ConnectionState.Connected), + ), ), ); @@ -663,7 +678,6 @@ export class CallViewModel extends ViewModel { // in a split-brained state. private readonly pretendToBeDisconnected$ = this.reconnecting$; - private readonly participants$ = this.scope.behavior< { participant: LocalParticipant | RemoteParticipant; @@ -731,7 +745,6 @@ export class CallViewModel extends ViewModel { // Handle room membership changes (and displayname updates) fromEvent(this.matrixRoom, RoomStateEvent.Members), // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), - ).pipe( startWith(null), map(() => { @@ -759,7 +772,7 @@ export class CallViewModel extends ViewModel { ); } return displaynameMap; - }, + }), ), ); @@ -971,21 +984,6 @@ export class CallViewModel extends ViewModel { this.memberships$.pipe(map((ms) => ms.length)), ); - private readonly allOthersLeft$ = this.memberships$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.every((m) => m.sender === this.userId) && - prev.some((m) => m.sender !== this.userId), - ), - map(() => {}), - take(1), - ); - - public readonly autoLeave$ = this.options.autoLeaveWhenOthersLeft - ? this.allOthersLeft$ - : NEVER; - private readonly didSendCallNotification$ = fromEvent( this.matrixRTCSession, MatrixRTCSessionEvent.DidSendCallNotification, @@ -994,6 +992,7 @@ export class CallViewModel extends ViewModel { MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification] > >; + /** * Whenever the RTC session tells us that it intends to ring the remote * participant's devices, this emits an Observable tracking the current state of @@ -1109,6 +1108,56 @@ export class CallViewModel extends ViewModel { map(() => {}), throttleTime(THROTTLE_SOUND_EFFECT_MS), ); + /** + * This observable tracks the matrix users that are currently in the call. + * There can be just one matrix user with multiple participants (see also participantChanges$) + */ + public readonly matrixUserChanges$ = this.userMedia$.pipe( + map( + (mediaItems) => + new Set( + mediaItems + .map((m) => m.vm.member?.userId) + .filter((id) => id !== undefined), + ), + ), + scan< + Set, + { + userIds: Set; + joinedUserIds: Set; + leftUserIds: Set; + } + >( + (prevState, userIds) => { + const left = new Set( + [...prevState.userIds].filter((id) => !userIds.has(id)), + ); + const joined = new Set( + [...userIds].filter((id) => !prevState.userIds.has(id)), + ); + return { userIds: userIds, joinedUserIds: joined, leftUserIds: left }; + }, + { userIds: new Set(), joinedUserIds: new Set(), leftUserIds: new Set() }, + ), + ); + + private readonly allOthersLeft$ = this.matrixUserChanges$.pipe( + map(({ userIds, leftUserIds }) => { + if (!this.userId) { + logger.warn("Could not access user ID to compute allOthersLeft"); + return false; + } + return ( + userIds.size === 1 && userIds.has(this.userId) && leftUserIds.size > 0 + ); + }), + startWith(false), + ); + + public readonly autoLeave$ = this.options.autoLeaveWhenOthersLeft + ? this.allOthersLeft$ + : NEVER; /** * List of MediaItems that we want to display, that are of type ScreenShare @@ -1791,8 +1840,6 @@ export class CallViewModel extends ViewModel { ), filter((v) => v.playSounds), ); - // TODO-REBASE: expose connection state observable - public readonly livekitConnectionState$: Observable; public constructor( // A call is permanently tied to a single Matrix room @@ -1839,18 +1886,34 @@ export class CallViewModel extends ViewModel { ); }); - this.join$.pipe(this.scope.bind()).subscribe(() => { - // TODO-MULTI-SFU: this makes no sense what so ever!!! - // need to look into this again. - // leaveRTCSession( - // this.matrixRTCSession, - // "user", // TODO-MULTI-SFU ? - // // Wait for the sound in widget mode (it's not long) - // Promise.resolve(), // TODO-MULTI-SFU - // //Promise.all([audioPromise, posthogRequest]), - // ).catch((e) => { - // logger.error("Error leaving RTC session", e); - // }); + this.allOthersLeft$ + .pipe( + this.scope.bind(), + filter((l) => (l && this.options.autoLeaveWhenOthersLeft) ?? false), + distinctUntilChanged(), + ) + .subscribe(() => { + this.leave$.next("allOthersLeft"); + }); + + this.callPickupState$.pipe(this.scope.bind()).subscribe((state) => { + if (state === "timeout" || state === "decline") { + this.leave$.next(state); + } + }); + + this.leave$.pipe(this.scope.bind()).subscribe((reason) => { + const { confineToRoom } = getUrlParams(); + leaveRTCSession(this.matrixRTCSession, "user") + // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts. + .then(() => { + if (!confineToRoom && !PosthogAnalytics.instance.isEnabled()) { + this._left$.next(reason); + } + }) + .catch((e) => { + logger.error("Error leaving RTC session", e); + }); }); // Pause upstream of all local media tracks when we're disconnected from From 41e152f4207d28be530feb9e895c134ac5a9f814 Mon Sep 17 00:00:00 2001 From: Timo K Date: Wed, 17 Sep 2025 11:25:49 +0200 Subject: [PATCH 25/45] dont throw disconnected error at start of the call Signed-off-by: Timo K --- src/room/InCallView.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index c12b44c9f..2f720148b 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -209,8 +209,9 @@ export const InCallView: FC = ({ // annoyingly we don't get the disconnection reason this way, // only by listening for the emitted event - if (connectionState === ConnectionState.Disconnected) - throw new ConnectionLostError(); + // This needs to be done differential. with the vm connection state we start with Disconnected. + // if (connectionState === ConnectionState.Disconnected) + // throw new ConnectionLostError(); const containerRef1 = useRef(null); const [containerRef2, bounds] = useMeasure(); From d9fe31039ff1bdbb2ffb4ea470e7d9289fe71afe Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 19 Sep 2025 18:01:45 +0200 Subject: [PATCH 26/45] start fixing CallViewModel tests. Signed-off-by: Timo K --- src/room/InCallView.tsx | 1 + src/room/VideoPreview.test.tsx | 13 +++---------- src/state/CallViewModel.test.ts | 12 +++--------- src/utils/test.ts | 8 ++++++++ 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 2f720148b..daf5034ac 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -210,6 +210,7 @@ export const InCallView: FC = ({ // annoyingly we don't get the disconnection reason this way, // only by listening for the emitted event // This needs to be done differential. with the vm connection state we start with Disconnected. + // TODO-MULTI-SFU decide how to handle this properly // if (connectionState === ConnectionState.Disconnected) // throw new ConnectionLostError(); diff --git a/src/room/VideoPreview.test.tsx b/src/room/VideoPreview.test.tsx index 3bbb6ad5f..717333eec 100644 --- a/src/room/VideoPreview.test.tsx +++ b/src/room/VideoPreview.test.tsx @@ -9,15 +9,8 @@ import { expect, describe, it, vi, beforeAll } from "vitest"; import { render } from "@testing-library/react"; import { type MatrixInfo, VideoPreview } from "./VideoPreview"; -import { type MuteStates } from "./MuteStates"; import { E2eeType } from "../e2ee/e2eeType"; - -function mockMuteStates({ audio = true, video = true } = {}): MuteStates { - return { - audio: { enabled: audio, setEnabled: vi.fn() }, - video: { enabled: video, setEnabled: vi.fn() }, - }; -} +import { mockMuteStates } from "../utils/test"; describe("VideoPreview", () => { const matrixInfo: MatrixInfo = { @@ -49,7 +42,7 @@ describe("VideoPreview", () => { const { queryByRole } = render( } />, @@ -61,7 +54,7 @@ describe("VideoPreview", () => { const { queryByRole } = render( } />, diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index ef4ef7625..b736b780e 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -53,7 +53,6 @@ import { type Layout, } from "./CallViewModel"; import { - mockLivekitRoom, mockLocalParticipant, mockMatrixRoom, mockMatrixRoomMember, @@ -62,6 +61,7 @@ import { mockRtcMembership, MockRTCSession, mockMediaDevices, + mockMuteStates, } from "../utils/test"; import { ECAddonConnectionState, @@ -340,21 +340,15 @@ function withCallViewModel( const roomEventSelectorSpy = vi .spyOn(ComponentsCore, "roomEventSelector") .mockImplementation((_room, _eventType) => of()); - - const livekitRoom = mockLivekitRoom( - { localParticipant }, - { remoteParticipants$ }, - ); - + const muteStates = mockMuteStates(); const raisedHands$ = new BehaviorSubject>({}); const vm = new CallViewModel( rtcSession as unknown as MatrixRTCSession, room, - livekitRoom, mediaDevices, + muteStates, options, - connectionState$, raisedHands$, new BehaviorSubject({}), ); diff --git a/src/utils/test.ts b/src/utils/test.ts index 31c6068a6..53b6d0ee4 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -53,6 +53,7 @@ import { Config } from "../config/Config"; import { type MediaDevices } from "../state/MediaDevices"; import { type Behavior, constant } from "../state/Behavior"; import { ObservableScope } from "../state/ObservableScope"; +import { Handler, MuteStates } from "../state/MuteStates"; export function withFakeTimers(continuation: () => void): void { vi.useFakeTimers(); @@ -417,3 +418,10 @@ export function mockMediaDevices(data: Partial): MediaDevices { ...data, } as MediaDevices; } + +export function mockMuteStates( + joined$: Observable = of(true), +): MuteStates { + const observableScope = new ObservableScope(); + return new MuteStates(observableScope, mockMediaDevices({}), joined$); +} From 02f23e25fd868bd1329667da3c714944b614f69f Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 22 Sep 2025 14:16:24 +0200 Subject: [PATCH 27/45] remove todo from matrix audio renderer Signed-off-by: Timo K --- src/livekit/MatrixAudioRenderer.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 8c4c9e103..0b87b27b5 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -81,8 +81,6 @@ export function LivekitRoomAudioRenderer({ loggedInvalidIdentities.current.add(identity); }; - // TODO-MULTI-SFU this uses the livekit room form the context. We need to change it so it uses the - // livekit room explicitly so we can pass a list of rooms into the audio renderer and call useTracks for each room. const tracks = useTracks( [ Track.Source.Microphone, From dddda7057f68922d663e92e7c2614b2767a7dd09 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 22 Sep 2025 14:17:38 +0200 Subject: [PATCH 28/45] add todo comments and who works on them Signed-off-by: Timo K --- src/room/GroupCallView.tsx | 1 + src/room/InCallView.tsx | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index ab083ffec..367f72a10 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -220,6 +220,7 @@ export const GroupCallView: FC = ({ try { setJoined(true); // TODO-MULTI-SFU what to do with error handling now that we don't use this function? + // @BillCarsonFr } catch (e) { if (e instanceof ElementCallError) { setExternalError(e); diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index daf5034ac..4daeae836 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -211,6 +211,7 @@ export const InCallView: FC = ({ // only by listening for the emitted event // This needs to be done differential. with the vm connection state we start with Disconnected. // TODO-MULTI-SFU decide how to handle this properly + // @BillCarsonFr // if (connectionState === ConnectionState.Disconnected) // throw new ConnectionLostError(); @@ -738,6 +739,7 @@ export const InCallView: FC = ({ const allLivekitRooms = useBehavior(vm.allLivekitRooms$); const memberships = useBehavior(vm.memberships$); const toggleScreensharing = useCallback(() => { + // TODO-MULTI-SFU implement screensharing throw new Error("TODO-MULTI-SFU"); // localParticipant // .setScreenShareEnabled(!isScreenShareEnabled, { From 8bf24895ce42554568eef258cb21af455c61e099 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 22 Sep 2025 14:18:23 +0200 Subject: [PATCH 29/45] TODO: settings modal with multiple connections Signed-off-by: Timo K --- src/room/InCallView.tsx | 2 +- src/settings/DeveloperSettingsTab.tsx | 33 +++++++++++++++------------ src/settings/SettingsModal.tsx | 12 +++++++--- src/state/CallViewModel.ts | 9 +++++++- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 4daeae836..b8460ad82 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -897,7 +897,7 @@ export const InCallView: FC = ({ onDismiss={closeSettings} tab={settingsTab} onTabChange={setSettingsTab} - livekitRoom={undefined} // TODO-MULTI-SFU + livekitRooms={allLivekitRooms} /> )} diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 36df5c39d..d503385b5 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -26,10 +26,10 @@ import styles from "./DeveloperSettingsTab.module.css"; import { useUrlParams } from "../UrlParams"; interface Props { client: MatrixClient; - livekitRoom?: LivekitRoom; + livekitRooms?: { room: LivekitRoom; url: string; isLocal?: boolean }[]; } -export const DeveloperSettingsTab: FC = ({ client, livekitRoom }) => { +export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { const { t } = useTranslation(); const [duplicateTiles, setDuplicateTiles] = useSetting(duplicateTilesSetting); const [debugTileLayout, setDebugTileLayout] = useSetting( @@ -59,15 +59,16 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRoom }) => { const urlParams = useUrlParams(); - const sfuUrl = useMemo((): URL | null => { - if (livekitRoom?.engine.client.ws?.url) { + const localSfuUrl = useMemo((): URL | null => { + const localRoom = livekitRooms?.find((r) => r.isLocal)?.room; + if (localRoom?.engine.client.ws?.url) { // strip the URL params - const url = new URL(livekitRoom.engine.client.ws.url); + const url = new URL(localRoom.engine.client.ws.url); url.search = ""; return url; } return null; - }, [livekitRoom]); + }, [livekitRooms]); return ( <> @@ -211,22 +212,26 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRoom }) => { )} />{" "} - {livekitRoom ? ( + {livekitRooms?.map((livekitRoom) => ( <> -

+

{t("developer_mode.livekit_sfu", { - url: sfuUrl?.href || "unknown", + url: livekitRoom.url || "unknown", })} +

+ {livekitRoom.isLocal &&

ws-url: {localSfuUrl?.href}

} +

+ {t("developer_mode.livekit_server_info")}( + {livekitRoom.isLocal ? "local" : "remote"})

-

{t("developer_mode.livekit_server_info")}

-            {livekitRoom.serverInfo
-              ? JSON.stringify(livekitRoom.serverInfo, null, 2)
+            {livekitRoom.room.serverInfo
+              ? JSON.stringify(livekitRoom.room.serverInfo, null, 2)
               : "undefined"}
-            {livekitRoom.metadata}
+            {livekitRoom.room.metadata}
- ) : null} + ))}

{t("developer_mode.environment_variables")}

{JSON.stringify(import.meta.env, null, 2)}

{t("developer_mode.url_params")}

diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 3272200df..9e5816476 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -51,7 +51,11 @@ interface Props { onTabChange: (tab: SettingsTab) => void; client: MatrixClient; roomId?: string; - livekitRoom?: LivekitRoom; + livekitRooms?: { + room: LivekitRoom; + url: string; + isLocal?: boolean; + }[]; } export const defaultSettingsTab: SettingsTab = "audio"; @@ -63,7 +67,7 @@ export const SettingsModal: FC = ({ onTabChange, client, roomId, - livekitRoom, + livekitRooms, }) => { const { t } = useTranslation(); @@ -204,7 +208,9 @@ export const SettingsModal: FC = ({ const developerTab: Tab = { key: "developer", name: t("settings.developer_tab_title"), - content: , + content: ( + + ), }; const tabs = [audioTab, videoTab]; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 6e922beef..8479f76ba 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -593,11 +593,18 @@ export class CallViewModel extends ViewModel { ]).pipe( map(([remoteConnections, localConnection, localFocus]) => Array.from(remoteConnections.entries()) - .map(([index, c]) => ({ room: c.livekitRoom, url: index })) + .map( + ([index, c]) => + ({ + room: c.livekitRoom, + url: index, + }) as { room: LivekitRoom; url: string; isLocal?: boolean }, + ) .concat([ { room: localConnection.livekitRoom, url: localFocus.livekit_service_url, + isLocal: true, }, ]), ), From 78e9521f22f6c428b009ff98d256be1b56ef4694 Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 23 Sep 2025 11:38:34 +0200 Subject: [PATCH 30/45] Make track processor work Signed-off-by: Timo K --- src/livekit/TrackProcessorContext.tsx | 42 ++++++++++++++++++++++++++- src/livekit/useLivekit.ts | 1 + src/room/InCallView.tsx | 4 +++ src/state/CallViewModel.ts | 3 ++ src/state/Connection.ts | 21 ++++++++++++-- 5 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/livekit/TrackProcessorContext.tsx b/src/livekit/TrackProcessorContext.tsx index b37a6e3e2..4a5ace46d 100644 --- a/src/livekit/TrackProcessorContext.tsx +++ b/src/livekit/TrackProcessorContext.tsx @@ -19,14 +19,21 @@ import { useMemo, } from "react"; import { type LocalVideoTrack } from "livekit-client"; +import { combineLatest, map, type Observable } from "rxjs"; +import { useObservable } from "observable-hooks"; import { backgroundBlur as backgroundBlurSettings, useSetting, } from "../settings/settings"; import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer"; +import { type Behavior } from "../state/Behavior"; -type ProcessorState = { +//TODO-MULTI-SFU: This is not yet fully there. +// it is a combination of exposing observable and react hooks. +// preferably we should not make this a context anymore and instead just a vm? + +export type ProcessorState = { supported: boolean | undefined; processor: undefined | ProcessorWrapper; }; @@ -42,6 +49,39 @@ export function useTrackProcessor(): ProcessorState { return state; } +export function useTrackProcessorObservable$(): Observable { + const state = use(ProcessorContext); + if (state === undefined) + throw new Error( + "useTrackProcessor must be used within a ProcessorProvider", + ); + const state$ = useObservable( + (init$) => init$.pipe(map(([init]) => init)), + [state], + ); + + return state$; +} + +export const trackProcessorSync = ( + videoTrack$: Behavior, + processor$: Behavior, +): void => { + combineLatest([videoTrack$, processor$]).subscribe( + ([videoTrack, processorState]) => { + if (!processorState) return; + if (!videoTrack) return; + const { processor } = processorState; + if (processor && !videoTrack.getProcessor()) { + void videoTrack.setProcessor(processor); + } + if (!processor && videoTrack.getProcessor()) { + void videoTrack.stopProcessor(); + } + }, + ); +}; + export const useTrackProcessorSync = ( videoTrack: LocalVideoTrack | null, ): void => { diff --git a/src/livekit/useLivekit.ts b/src/livekit/useLivekit.ts index 0672a8eb8..420bac95f 100644 --- a/src/livekit/useLivekit.ts +++ b/src/livekit/useLivekit.ts @@ -55,6 +55,7 @@ interface UseLivekitResult { } // TODO-MULTI-SFU This is not used anymore but the device syncing logic needs to be moved into the connection object. +// seems to be mostly done... See Connection.ts export function useLivekitPublicationRoom( rtcSession: MatrixRTCSession, muteStates: MuteStates, diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index b8460ad82..157ee46a3 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -114,6 +114,7 @@ import { useAudioContext } from "../useAudioContext"; import ringtoneMp3 from "../sound/ringtone.mp3?url"; import ringtoneOgg from "../sound/ringtone.ogg?url"; import { ConnectionLostError } from "../utils/errors.ts"; +import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.tsx"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -133,6 +134,7 @@ export const ActiveCall: FC = (props) => { const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = useUrlParams(); + const trackProcessorState$ = useTrackProcessorObservable$(); useEffect(() => { const reactionsReader = new ReactionsReader(props.rtcSession); const vm = new CallViewModel( @@ -147,6 +149,7 @@ export const ActiveCall: FC = (props) => { }, reactionsReader.raisedHands$, reactionsReader.reactions$, + trackProcessorState$, ); setVm(vm); @@ -166,6 +169,7 @@ export const ActiveCall: FC = (props) => { sendNotificationType, waitForCallPickup, props.onLeft, + trackProcessorState$, ]); if (vm === null) return null; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 8479f76ba..408283574 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -125,6 +125,7 @@ import { Connection, PublishConnection } from "./Connection"; import { type MuteStates } from "./MuteStates"; import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; import { getUrlParams } from "../UrlParams"; +import { type ProcessorState } from "../livekit/TrackProcessorContext"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -460,6 +461,7 @@ export class CallViewModel extends ViewModel { this.mediaDevices, this.muteStates, this.e2eeLivekitOptions(), + this.scope.behavior(this.trackProcessorState$), ), ); @@ -1861,6 +1863,7 @@ export class CallViewModel extends ViewModel { private readonly reactionsSubject$: Observable< Record >, + private readonly trackProcessorState$: Observable, ) { super(); diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 4f9721eaf..6804b2b7b 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -15,6 +15,7 @@ import { Room as LivekitRoom, type E2EEOptions, Track, + LocalVideoTrack, } from "livekit-client"; import { type MatrixClient } from "matrix-js-sdk"; import { @@ -39,6 +40,11 @@ import { defaultLiveKitOptions } from "../livekit/options"; import { getValue } from "../utils/observable"; import { getUrlParams } from "../UrlParams"; import { type MuteStates } from "./MuteStates"; +import { + type ProcessorState, + trackProcessorSync, +} from "../livekit/TrackProcessorContext"; +import { observeTrackReference$ } from "./MediaViewModel"; export class Connection { protected stopped = false; @@ -151,6 +157,7 @@ export class PublishConnection extends Connection { devices: MediaDevices, private readonly muteStates: MuteStates, e2eeLivekitOptions: E2EEOptions | undefined, + trackerProcessorState$: Behavior, ) { logger.info("[LivekitRoom] Create LiveKit room"); const { controlledAudioDevices } = getUrlParams(); @@ -160,8 +167,7 @@ export class PublishConnection extends Connection { videoCaptureDefaults: { ...defaultLiveKitOptions.videoCaptureDefaults, deviceId: devices.videoInput.selected$.value?.id, - // TODO-MULTI-SFU add processor support back - // processor, + processor: trackerProcessorState$.value.processor, }, audioCaptureDefaults: { ...defaultLiveKitOptions.audioCaptureDefaults, @@ -191,6 +197,17 @@ export class PublishConnection extends Connection { room, ); + // Setup track processor syncing (blur) + const track$ = this.scope.behavior( + observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( + map((trackRef) => { + const track = trackRef?.publication?.track; + return track instanceof LocalVideoTrack ? track : null; + }), + ), + ); + trackProcessorSync(track$, trackerProcessorState$); + this.muteStates.audio.setHandler(async (desired) => { try { await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); From 7777179935f83cc0c2e79663c7a3ed3036b89a98 Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 23 Sep 2025 11:40:29 +0200 Subject: [PATCH 31/45] cleanup (delete files useLivekit) now covered by Connection.ts Signed-off-by: Timo K --- src/livekit/livekitSubscriptionRoom.ts | 123 ------- src/livekit/useLivekit.ts | 441 ------------------------- src/state/Connection.ts | 2 - 3 files changed, 566 deletions(-) delete mode 100644 src/livekit/livekitSubscriptionRoom.ts delete mode 100644 src/livekit/useLivekit.ts diff --git a/src/livekit/livekitSubscriptionRoom.ts b/src/livekit/livekitSubscriptionRoom.ts deleted file mode 100644 index f92ff10e5..000000000 --- a/src/livekit/livekitSubscriptionRoom.ts +++ /dev/null @@ -1,123 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - ConnectionState, - type E2EEManagerOptions, - ExternalE2EEKeyProvider, - LocalVideoTrack, - Room, - type RoomOptions, -} from "livekit-client"; -import { useEffect, useRef } from "react"; -import E2EEWorker from "livekit-client/e2ee-worker?worker"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; - -import { defaultLiveKitOptions } from "./options"; -import { type SFUConfig } from "./openIDSFU"; -import { type MuteStates } from "../room/MuteStates"; -import { useMediaDevices } from "../MediaDevicesContext"; -import { - type ECConnectionState, - useECConnectionState, -} from "./useECConnectionState"; -import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { E2eeType } from "../e2ee/e2eeType"; -import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; -import { - useTrackProcessor, - useTrackProcessorSync, -} from "./TrackProcessorContext"; -import { observeTrackReference$ } from "../state/MediaViewModel"; -import { useUrlParams } from "../UrlParams"; -import { useInitial } from "../useInitial"; -import { getValue } from "../utils/observable"; -import { type SelectedDevice } from "../state/MediaDevices"; - -interface UseLivekitResult { - livekitPublicationRoom?: Room; - connState: ECConnectionState; -} - -// TODO-MULTI-SFU This is all the logic we need in the subscription connection logic (sync output devices) -// This is not used! (but summarizes what we need) -export function livekitSubscriptionRoom( - rtcSession: MatrixRTCSession, - muteStates: MuteStates, - sfuConfig: SFUConfig | undefined, - e2eeSystem: EncryptionSystem, -): UseLivekitResult { - // Only ever create the room once via useInitial. - // The call can end up with multiple livekit rooms. This is the particular room in - // which this participant publishes their media. - const publicationRoom = useInitial(() => { - logger.info("[LivekitRoom] Create LiveKit room"); - - let e2ee: E2EEManagerOptions | undefined; - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - logger.info("Created MatrixKeyProvider (per participant)"); - e2ee = { - keyProvider: new MatrixKeyProvider(), - worker: new E2EEWorker(), - }; - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - logger.info("Created ExternalE2EEKeyProvider (shared key)"); - e2ee = { - keyProvider: new ExternalE2EEKeyProvider(), - worker: new E2EEWorker(), - }; - } - - const roomOptions: RoomOptions = { - ...defaultLiveKitOptions, - audioOutput: { - // When using controlled audio devices, we don't want to set the - // deviceId here, because it will be set by the native app. - // (also the id does not need to match a browser device id) - deviceId: controlledAudioDevices - ? undefined - : getValue(devices.audioOutput.selected$)?.id, - }, - e2ee, - }; - // We have to create the room manually here due to a bug inside - // @livekit/components-react. JSON.stringify() is used in deps of a - // useEffect() with an argument that references itself, if E2EE is enabled - const room = new Room(roomOptions); - room.setE2EEEnabled(e2eeSystem.kind !== E2eeType.NONE).catch((e) => { - logger.error("Failed to set E2EE enabled on room", e); - }); - - return room; - }); - - // Setup and update the keyProvider which was create by `createRoom` - useEffect(() => { - const e2eeOptions = publicationRoom.options.e2ee; - if ( - e2eeSystem.kind === E2eeType.NONE || - !(e2eeOptions && "keyProvider" in e2eeOptions) - ) - return; - - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - (e2eeOptions.keyProvider as MatrixKeyProvider).setRTCSession(rtcSession); - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - (e2eeOptions.keyProvider as ExternalE2EEKeyProvider) - .setKey(e2eeSystem.secret) - .catch((e) => { - logger.error("Failed to set shared key for E2EE", e); - }); - } - }, [publicationRoom.options.e2ee, e2eeSystem, rtcSession]); - - return { - connState: connectionState, - livekitPublicationRoom: publicationRoom, - }; -} diff --git a/src/livekit/useLivekit.ts b/src/livekit/useLivekit.ts deleted file mode 100644 index 420bac95f..000000000 --- a/src/livekit/useLivekit.ts +++ /dev/null @@ -1,441 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - ConnectionState, - type E2EEManagerOptions, - ExternalE2EEKeyProvider, - type LocalTrackPublication, - LocalVideoTrack, - Room, - type RoomOptions, - Track, -} from "livekit-client"; -import { useEffect, useRef } from "react"; -import E2EEWorker from "livekit-client/e2ee-worker?worker"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; -import { useObservable, useObservableEagerState } from "observable-hooks"; -import { - map, - NEVER, - type Observable, - type Subscription, - switchMap, -} from "rxjs"; - -import { defaultLiveKitOptions } from "./options"; -import { type SFUConfig } from "./openIDSFU"; -import { type MuteStates } from "../room/MuteStates"; -import { useMediaDevices } from "../MediaDevicesContext"; -import { - type ECConnectionState, - useECConnectionState, -} from "./useECConnectionState"; -import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { E2eeType } from "../e2ee/e2eeType"; -import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; -import { - useTrackProcessor, - useTrackProcessorSync, -} from "./TrackProcessorContext"; -import { observeTrackReference$ } from "../state/MediaViewModel"; -import { useUrlParams } from "../UrlParams"; -import { useInitial } from "../useInitial"; -import { getValue } from "../utils/observable"; -import { type SelectedDevice } from "../state/MediaDevices"; - -interface UseLivekitResult { - livekitPublicationRoom?: Room; - connState: ECConnectionState; -} - -// TODO-MULTI-SFU This is not used anymore but the device syncing logic needs to be moved into the connection object. -// seems to be mostly done... See Connection.ts -export function useLivekitPublicationRoom( - rtcSession: MatrixRTCSession, - muteStates: MuteStates, - sfuConfig: SFUConfig | undefined, - e2eeSystem: EncryptionSystem, -): UseLivekitResult { - const { controlledAudioDevices } = useUrlParams(); - - const initialMuteStates = useInitial(() => muteStates); - - const devices = useMediaDevices(); - const initialAudioInputId = useInitial( - () => getValue(devices.audioInput.selected$)?.id, - ); - - // Store if audio/video are currently updating. If to prohibit unnecessary calls - // to setMicrophoneEnabled/setCameraEnabled - const audioMuteUpdating = useRef(false); - const videoMuteUpdating = useRef(false); - // Store the current button mute state that gets passed to this hook via props. - // We need to store it for awaited code that relies on the current value. - const buttonEnabled = useRef({ - audio: initialMuteStates.audio.enabled, - video: initialMuteStates.video.enabled, - }); - - const { processor } = useTrackProcessor(); - - // Only ever create the room once via useInitial. - // The call can end up with multiple livekit rooms. This is the particular room in - // which this participant publishes their media. - const publicationRoom = useInitial(() => { - logger.info("[LivekitRoom] Create LiveKit room"); - - let e2ee: E2EEManagerOptions | undefined; - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - logger.info("Created MatrixKeyProvider (per participant)"); - e2ee = { - keyProvider: new MatrixKeyProvider(), - worker: new E2EEWorker(), - }; - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - logger.info("Created ExternalE2EEKeyProvider (shared key)"); - e2ee = { - keyProvider: new ExternalE2EEKeyProvider(), - worker: new E2EEWorker(), - }; - } - - const roomOptions: RoomOptions = { - ...defaultLiveKitOptions, - videoCaptureDefaults: { - ...defaultLiveKitOptions.videoCaptureDefaults, - deviceId: getValue(devices.videoInput.selected$)?.id, - processor, - }, - audioCaptureDefaults: { - ...defaultLiveKitOptions.audioCaptureDefaults, - deviceId: initialAudioInputId, - }, - audioOutput: { - // When using controlled audio devices, we don't want to set the - // deviceId here, because it will be set by the native app. - // (also the id does not need to match a browser device id) - deviceId: controlledAudioDevices - ? undefined - : getValue(devices.audioOutput.selected$)?.id, - }, - e2ee, - }; - // We have to create the room manually here due to a bug inside - // @livekit/components-react. JSON.stringify() is used in deps of a - // useEffect() with an argument that references itself, if E2EE is enabled - const room = new Room(roomOptions); - room.setE2EEEnabled(e2eeSystem.kind !== E2eeType.NONE).catch((e) => { - logger.error("Failed to set E2EE enabled on room", e); - }); - - return room; - }); - - // Setup and update the keyProvider which was create by `createRoom` - useEffect(() => { - const e2eeOptions = publicationRoom.options.e2ee; - if ( - e2eeSystem.kind === E2eeType.NONE || - !(e2eeOptions && "keyProvider" in e2eeOptions) - ) - return; - - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - (e2eeOptions.keyProvider as MatrixKeyProvider).setRTCSession(rtcSession); - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - (e2eeOptions.keyProvider as ExternalE2EEKeyProvider) - .setKey(e2eeSystem.secret) - .catch((e) => { - logger.error("Failed to set shared key for E2EE", e); - }); - } - }, [publicationRoom.options.e2ee, e2eeSystem, rtcSession]); - - // Sync the requested track processors with LiveKit - useTrackProcessorSync( - useObservableEagerState( - useObservable( - (room$) => - room$.pipe( - switchMap(([room]) => - observeTrackReference$( - room.localParticipant, - Track.Source.Camera, - ), - ), - map((trackRef) => { - const track = trackRef?.publication?.track; - return track instanceof LocalVideoTrack ? track : null; - }), - ), - [publicationRoom], - ), - ), - ); - - const connectionState = useECConnectionState( - initialAudioInputId, - initialMuteStates.audio.enabled, - publicationRoom, - sfuConfig, - ); - - // Log errors when local participant has issues publishing a track. - useEffect(() => { - const localTrackUnpublishedFn = ( - publication: LocalTrackPublication, - ): void => { - logger.info( - "Local track unpublished", - publication.trackName, - publication.trackInfo, - ); - }; - const mediaDevicesErrorFn = (error: Error): void => { - logger.warn("Media devices error when publishing a track", error); - }; - - room.localParticipant.on("localTrackUnpublished", localTrackUnpublishedFn); - room.localParticipant.on("mediaDevicesError", mediaDevicesErrorFn); - - return (): void => { - room.localParticipant.off( - "localTrackUnpublished", - localTrackUnpublishedFn, - ); - room.localParticipant.off("mediaDevicesError", mediaDevicesErrorFn); - }; - }, [room.localParticipant]); - - useEffect(() => { - // Sync the requested mute states with LiveKit's mute states. We do it this - // way around rather than using LiveKit as the source of truth, so that the - // states can be consistent throughout the lobby and loading screens. - // It's important that we only do this in the connected state, because - // LiveKit's internal mute states aren't consistent during connection setup, - // and setting tracks to be enabled during this time causes errors. - if ( - publicationRoom !== undefined && - connectionState === ConnectionState.Connected - ) { - const participant = publicationRoom.localParticipant; - // Always update the muteButtonState Ref so that we can read the current - // state in awaited blocks. - buttonEnabled.current = { - audio: muteStates.audio.enabled, - video: muteStates.video.enabled, - }; - - enum MuteDevice { - Microphone, - Camera, - } - - const syncMuteState = async ( - iterCount: number, - type: MuteDevice, - ): Promise => { - // The approach for muting is to always bring the actual livekit state in sync with the button - // This allows for a very predictable and reactive behavior for the user. - // (the new state is the old state when pressing the button n times (where n is even)) - // (the new state is different to the old state when pressing the button n times (where n is uneven)) - // In case there are issues with the device there might be situations where setMicrophoneEnabled/setCameraEnabled - // return immediately. This should be caught with the Error("track with new mute state could not be published"). - // For now we are still using an iterCount to limit the recursion loop to 10. - // This could happen if the device just really does not want to turn on (hardware based issue) - // but the mute button is in unmute state. - // For now our fail mode is to just stay in this state. - // TODO: decide for a UX on how that fail mode should be treated (disable button, hide button, sync button back to muted without user input) - - if (iterCount > 10) { - logger.error( - "Stop trying to sync the input device with current mute state after 10 failed tries", - ); - return; - } - let devEnabled; - let btnEnabled; - let updating; - switch (type) { - case MuteDevice.Microphone: - devEnabled = participant.isMicrophoneEnabled; - btnEnabled = buttonEnabled.current.audio; - updating = audioMuteUpdating.current; - break; - case MuteDevice.Camera: - devEnabled = participant.isCameraEnabled; - btnEnabled = buttonEnabled.current.video; - updating = videoMuteUpdating.current; - break; - } - if (devEnabled !== btnEnabled && !updating) { - try { - let trackPublication; - switch (type) { - case MuteDevice.Microphone: - audioMuteUpdating.current = true; - trackPublication = await participant.setMicrophoneEnabled( - buttonEnabled.current.audio, - publicationRoom.options.audioCaptureDefaults, - ); - audioMuteUpdating.current = false; - break; - case MuteDevice.Camera: - videoMuteUpdating.current = true; - trackPublication = await participant.setCameraEnabled( - buttonEnabled.current.video, - publicationRoom.options.videoCaptureDefaults, - ); - videoMuteUpdating.current = false; - break; - } - - if (trackPublication) { - // await participant.setMicrophoneEnabled can return immediately in some instances, - // so that participant.isMicrophoneEnabled !== buttonEnabled.current.audio still holds true. - // This happens if the device is still in a pending state - // "sleeping" here makes sure we let react do its thing so that participant.isMicrophoneEnabled is updated, - // so we do not end up in a recursion loop. - await new Promise((r) => setTimeout(r, 100)); - - // track got successfully changed to mute/unmute - // Run the check again after the change is done. Because the user - // can update the state (presses mute button) while the device is enabling - // itself we need might need to update the mute state right away. - // This async recursion makes sure that setCamera/MicrophoneEnabled is - // called as little times as possible. - await syncMuteState(iterCount + 1, type); - } else { - throw new Error( - "track with new mute state could not be published", - ); - } - } catch (e) { - if ((e as DOMException).name === "NotAllowedError") { - logger.error( - "Fatal error while syncing mute state: resetting", - e, - ); - if (type === MuteDevice.Microphone) { - audioMuteUpdating.current = false; - muteStates.audio.setEnabled?.(false); - } else { - videoMuteUpdating.current = false; - muteStates.video.setEnabled?.(false); - } - } else { - logger.error( - "Failed to sync audio mute state with LiveKit (will retry to sync in 1s):", - e, - ); - setTimeout(() => { - syncMuteState(iterCount + 1, type).catch((e) => { - logger.error( - `Failed to sync ${MuteDevice[type]} mute state with LiveKit iterCount=${iterCount + 1}`, - e, - ); - }); - }, 1000); - } - } - } - }; - - syncMuteState(0, MuteDevice.Microphone).catch((e) => { - logger.error("Failed to sync audio mute state with LiveKit", e); - }); - syncMuteState(0, MuteDevice.Camera).catch((e) => { - logger.error("Failed to sync video mute state with LiveKit", e); - }); - } - }, [publicationRoom, muteStates, connectionState]); - - useEffect(() => { - // Sync the requested devices with LiveKit's devices - if ( - publicationRoom !== undefined && - connectionState === ConnectionState.Connected - ) { - const syncDevice = ( - kind: MediaDeviceKind, - selected$: Observable, - ): Subscription => - selected$.subscribe((device) => { - logger.info( - "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", - publicationRoom.getActiveDevice(kind), - " !== ", - device?.id, - ); - if ( - device !== undefined && - publicationRoom.getActiveDevice(kind) !== device.id - ) { - publicationRoom - .switchActiveDevice(kind, device.id) - .catch((e) => - logger.error(`Failed to sync ${kind} device with LiveKit`, e), - ); - } - }); - - const subscriptions = [ - syncDevice("audioinput", devices.audioInput.selected$), - !controlledAudioDevices - ? syncDevice("audiooutput", devices.audioOutput.selected$) - : undefined, - syncDevice("videoinput", devices.videoInput.selected$), - // Restart the audio input track whenever we detect that the active media - // device has changed to refer to a different hardware device. We do this - // for the sake of Chrome, which provides a "default" device that is meant - // to match the system's default audio input, whatever that may be. - // This is special-cased for only audio inputs because we need to dig around - // in the LocalParticipant object for the track object and there's not a nice - // way to do that generically. There is usually no OS-level default video capture - // device anyway, and audio outputs work differently. - devices.audioInput.selected$ - .pipe(switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER)) - .subscribe(() => { - const activeMicTrack = Array.from( - publicationRoom.localParticipant.audioTrackPublications.values(), - ).find((d) => d.source === Track.Source.Microphone)?.track; - - if ( - activeMicTrack && - // only restart if the stream is still running: LiveKit will detect - // when a track stops & restart appropriately, so this is not our job. - // Plus, we need to avoid restarting again if the track is already in - // the process of being restarted. - activeMicTrack.mediaStreamTrack.readyState !== "ended" - ) { - // Restart the track, which will cause Livekit to do another - // getUserMedia() call with deviceId: default to get the *new* default device. - // Note that room.switchActiveDevice() won't work: Livekit will ignore it because - // the deviceId hasn't changed (was & still is default). - publicationRoom.localParticipant - .getTrackPublication(Track.Source.Microphone) - ?.audioTrack?.restartTrack() - .catch((e) => { - logger.error(`Failed to restart audio device track`, e); - }); - } - }), - ]; - - return (): void => { - for (const s of subscriptions) s?.unsubscribe(); - }; - } - }, [publicationRoom, devices, connectionState, controlledAudioDevices]); - - return { - connState: connectionState, - livekitPublicationRoom: publicationRoom, - }; -} diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 6804b2b7b..e14ddc9a9 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -1,4 +1,3 @@ -// TODO-MULTI-SFU Add all device syncing logic from useLivekit /* Copyright 2025 New Vector Ltd. @@ -294,5 +293,4 @@ export class PublishConnection extends Connection { } }); } - // TODO-MULTI-SFU Sync the requested track processors with LiveKit } From 96e96a5e43b9d6a6b8b2b0a3428a5fe6dc82eeab Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 23 Sep 2025 12:25:05 +0200 Subject: [PATCH 32/45] fix leaving Signed-off-by: Timo K --- src/room/GroupCallView.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 367f72a10..614c7b50b 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -68,7 +68,6 @@ import { import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary.tsx"; import { useNewMembershipManager as useNewMembershipManagerSetting, - useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting, useSetting, } from "../settings/settings"; import { useTypedEventEmitter } from "../useEvents"; @@ -310,6 +309,7 @@ export const GroupCallView: FC = ({ useNewMembershipManager, ]); + // TODO refactor this + "joined" to just one callState const [left, setLeft] = useState(false); const navigate = useNavigate(); @@ -319,6 +319,7 @@ export const GroupCallView: FC = ({ let playSound: CallEventSounds = "left"; if (reason === "timeout" || reason === "decline") playSound = reason; + setJoined(false); setLeft(true); const audioPromise = leaveSoundContext.current?.playSound(playSound); // In embedded/widget mode the iFrame will be killed right after the call ended prohibiting the posthog event from getting sent, @@ -354,6 +355,7 @@ export const GroupCallView: FC = ({ ); }, [ + setJoined, leaveSoundContext, widget, room.roomId, From 6b44f3b0081ad473603cfeb82d04c8b276bd41ae Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 23 Sep 2025 12:25:31 +0200 Subject: [PATCH 33/45] a tiny bit of tests lint fixes. Signed-off-by: Timo K --- src/utils/test-viewmodel.ts | 22 ++++++---------------- src/utils/test.ts | 2 +- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 687adba79..785cbe1ba 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -5,7 +5,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { ConnectionState } from "livekit-client"; import { type CallMembership, type MatrixRTCSession, @@ -27,19 +26,13 @@ import { type CallViewModelOptions, } from "../state/CallViewModel"; import { - mockLivekitRoom, mockMatrixRoom, mockMediaDevices, + mockMuteStates, MockRTCSession, } from "./test"; -import { - aliceRtcMember, - aliceParticipant, - localParticipant, - localRtcMember, -} from "./test-fixtures"; +import { aliceRtcMember, localRtcMember } from "./test-fixtures"; import { type RaisedHandInfo, type ReactionInfo } from "../reactions"; -import { constant } from "../state/Behavior"; export function getBasicRTCSession( members: RoomMember[], @@ -141,23 +134,20 @@ export function getBasicCallViewModelEnvironment( const handRaisedSubject$ = new BehaviorSubject({}); const reactionsSubject$ = new BehaviorSubject({}); - const remoteParticipants$ = of([aliceParticipant]); - const livekitRoom = mockLivekitRoom( - { localParticipant }, - { remoteParticipants$ }, - ); + // const remoteParticipants$ = of([aliceParticipant]); + const vm = new CallViewModel( rtcSession as unknown as MatrixRTCSession, matrixRoom, - livekitRoom, mockMediaDevices({}), + mockMuteStates(), { encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, ...callViewModelOptions, }, - constant(ConnectionState.Connected), handRaisedSubject$, reactionsSubject$, + of({ processor: undefined, supported: false }), ); return { vm, diff --git a/src/utils/test.ts b/src/utils/test.ts index 53b6d0ee4..4aebd5fdb 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -53,7 +53,7 @@ import { Config } from "../config/Config"; import { type MediaDevices } from "../state/MediaDevices"; import { type Behavior, constant } from "../state/Behavior"; import { ObservableScope } from "../state/ObservableScope"; -import { Handler, MuteStates } from "../state/MuteStates"; +import { MuteStates } from "../state/MuteStates"; export function withFakeTimers(continuation: () => void): void { vi.useFakeTimers(); From f99a256c8616eba681bcb8fc0954bfd9fcc2fbef Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 24 Sep 2025 13:53:39 -0400 Subject: [PATCH 34/45] Reset matrix-js-sdk to multi SFU branch --- package.json | 2 +- yarn.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 18877823f..915830233 100644 --- a/package.json +++ b/package.json @@ -108,7 +108,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=develop", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU", "matrix-widget-api": "^1.13.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/yarn.lock b/yarn.lock index 668706b4b..6d0feed38 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7544,7 +7544,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=develop" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU" matrix-widget-api: "npm:^1.13.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -10297,9 +10297,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU": version: 37.13.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=2f1d654f14be8dd03896e9e76f12017b6f9eec1c" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=b61e39a81458fb02d76d384e9c4bbef30fcd516a" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.1.0" @@ -10315,9 +10315,9 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:11" - checksum: 10c0/ecd019c677c272c5598617dcde407dbe4b1b11460863b2a577e33f3fd8732c9d9073ec0221b471ec1eb24e2839eec20728db7f92c9348be83126547286e50805 + checksum: 10c0/da28671be560d3ef56e5a5465d1793dd6c2ccefcf68b510726f21b0a62107b115d40c1940e2a9bb8b915abdc600a9a84b248175ccdd9c77a3b313733efb8f497 languageName: node - linkType: soft + linkType: hard "matrix-widget-api@npm:^1.10.0, matrix-widget-api@npm:^1.13.0": version: 1.13.1 From edd3eb874786e5cbc1ffe4171745cbfc4ed45066 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 24 Sep 2025 13:54:54 -0400 Subject: [PATCH 35/45] Implement screen sharing --- src/room/InCallView.tsx | 23 ++++----------- src/state/CallViewModel.ts | 59 +++++++++++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 31 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 157ee46a3..14f18fb7a 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -116,8 +116,6 @@ import ringtoneOgg from "../sound/ringtone.ogg?url"; import { ConnectionLostError } from "../utils/errors.ts"; import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.tsx"; -const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); - const maxTapDurationMs = 400; export interface ActiveCallProps @@ -224,7 +222,7 @@ export const InCallView: FC = ({ // Merge the refs so they can attach to the same element const containerRef = useMergedRefs(containerRef1, containerRef2); - const { hideScreensharing, showControls } = useUrlParams(); + const { showControls } = useUrlParams(); const muteAllAudio = useBehavior(muteAllAudio$); // Call pickup state and display names are needed for waiting overlay/sounds @@ -299,6 +297,7 @@ export const InCallView: FC = ({ const showFooter = useBehavior(vm.showFooter$); const earpieceMode = useBehavior(vm.earpieceMode$); const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$); + const sharingScreen = useBehavior(vm.sharingScreen$); // We need to set the proper timings on the animation based upon the sound length. const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1; @@ -742,18 +741,6 @@ export const InCallView: FC = ({ const allLivekitRooms = useBehavior(vm.allLivekitRooms$); const memberships = useBehavior(vm.memberships$); - const toggleScreensharing = useCallback(() => { - // TODO-MULTI-SFU implement screensharing - throw new Error("TODO-MULTI-SFU"); - // localParticipant - // .setScreenShareEnabled(!isScreenShareEnabled, { - // audio: true, - // selfBrowserSurface: "include", - // surfaceSwitching: "include", - // systemAudio: "include", - // }) - // .catch(logger.error); - }, []); const buttons: JSX.Element[] = []; @@ -775,13 +762,13 @@ export const InCallView: FC = ({ data-testid="incall_videomute" />, ); - if (canScreenshare && !hideScreensharing) { + if (vm.toggleScreenSharing !== null) { buttons.push( , diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 408283574..af750a9bd 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -15,6 +15,7 @@ import { type LocalParticipant, ParticipantEvent, type RemoteParticipant, + type Participant, } from "livekit-client"; import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { @@ -341,18 +342,7 @@ class UserMedia { this.presenter$ = this.scope.behavior( this.participant$.pipe( - switchMap( - (p) => - (p && - observeParticipantEvents( - p, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe(map((p) => p.isScreenShareEnabled))) ?? - of(false), - ), + switchMap((p) => (p === undefined ? of(false) : sharingScreen$(p))), ), ); } @@ -433,7 +423,19 @@ function getRoomMemberFromRtcMember( return { id, member }; } +function sharingScreen$(p: Participant): Observable { + return observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe(map((p) => p.isScreenShareEnabled)); +} + export class CallViewModel extends ViewModel { + private readonly urlParams = getUrlParams(); + private readonly livekitAlias = getLivekitAlias(this.matrixRTCSession); private readonly livekitE2EEKeyProvider = getE2eeKeyProvider( @@ -1850,6 +1852,37 @@ export class CallViewModel extends ViewModel { filter((v) => v.playSounds), ); + /** + * Whether we are sharing our screen. + */ + public readonly sharingScreen$ = this.scope.behavior( + from(this.localConnection).pipe( + switchMap((c) => sharingScreen$(c.livekitRoom.localParticipant)), + startWith(false), + ), + ); + + /** + * Callback for toggling screen sharing. If null, screen sharing is not + * available. + */ + public readonly toggleScreenSharing = + "getDisplayMedia" in (navigator.mediaDevices ?? {}) && + !this.urlParams.hideScreensharing + ? (): void => + void this.localConnection.then( + (c) => + void c.livekitRoom.localParticipant + .setScreenShareEnabled(!this.sharingScreen$.value, { + audio: true, + selfBrowserSurface: "include", + surfaceSwitching: "include", + systemAudio: "include", + }) + .catch(logger.error), + ) + : null; + public constructor( // A call is permanently tied to a single Matrix room private readonly matrixRTCSession: MatrixRTCSession, @@ -1913,7 +1946,7 @@ export class CallViewModel extends ViewModel { }); this.leave$.pipe(this.scope.bind()).subscribe((reason) => { - const { confineToRoom } = getUrlParams(); + const { confineToRoom } = this.urlParams; leaveRTCSession(this.matrixRTCSession, "user") // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts. .then(() => { From 6cf020763e4c1ad23f0fda75ec9e7f7cc9d72d98 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 24 Sep 2025 21:26:16 -0400 Subject: [PATCH 36/45] Make UI react instantly to hanging up but also wait for leave sound This ensures that we don't see a mistaken 'reconnecting' toast while we're hanging up (and also that the leave sound gets a chance to play in widgets once again). --- src/room/GroupCallView.tsx | 76 ++++++++++++++++++------------ src/room/InCallView.tsx | 4 +- src/state/CallViewModel.ts | 95 +++++++++++++++++++------------------- 3 files changed, 95 insertions(+), 80 deletions(-) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 614c7b50b..49d8b60b0 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -41,7 +41,6 @@ import { ActiveCall } from "./InCallView"; import { type MuteStates } from "../state/MuteStates"; import { useMediaDevices } from "../MediaDevicesContext"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; -import { leaveRTCSession } from "../rtcSessionHelpers"; import { saveKeyForRoom, useRoomEncryptionSystem, @@ -50,7 +49,12 @@ import { useRoomAvatar } from "./useRoomAvatar"; import { useRoomName } from "./useRoomName"; import { useJoinRule } from "./useJoinRule"; import { InviteModal } from "./InviteModal"; -import { HeaderStyle, type UrlParams, useUrlParams } from "../UrlParams"; +import { + getUrlParams, + HeaderStyle, + type UrlParams, + useUrlParams, +} from "../UrlParams"; import { E2eeType } from "../e2ee/e2eeType"; import { useAudioContext } from "../useAudioContext"; import { @@ -322,37 +326,62 @@ export const GroupCallView: FC = ({ setJoined(false); setLeft(true); const audioPromise = leaveSoundContext.current?.playSound(playSound); - // In embedded/widget mode the iFrame will be killed right after the call ended prohibiting the posthog event from getting sent, - // therefore we want the event to be sent instantly without getting queued/batched. - const sendInstantly = !!widget; - // we need to wait until the callEnded event is tracked on posthog. - // Otherwise the iFrame gets killed before the callEnded event got tracked. + // We need to wait until the callEnded event is tracked on PostHog, + // otherwise the iframe may get killed first. const posthogRequest = new Promise((resolve) => { + // To increase the likelihood of the PostHog event being sent out in + // widget mode before the iframe is killed, we ask it to skip the + // usual queuing/batching of requests. + const sendInstantly = widget !== null; PosthogAnalytics.instance.eventCallEnded.track( room.roomId, rtcSession.memberships.length, sendInstantly, - rtcSession, ); + // Unfortunately the PostHog library provides no way to await the + // tracking of an event, but we don't really want it to hold up the + // closing of the widget that long anyway, so giving it 10 ms will do. window.setTimeout(resolve, 10); }); void Promise.all([audioPromise, posthogRequest]) - .then(() => { + .catch((e) => + logger.error( + "Failed to play leave audio and/or send PostHog leave event", + e, + ), + ) + .then(async () => { if ( !isPasswordlessUser && !confineToRoom && !PosthogAnalytics.instance.isEnabled() - ) { + ) void navigate("/"); + + if (widget) { + // After this point the iframe could die at any moment! + try { + await widget.api.setAlwaysOnScreen(false); + } catch (e) { + logger.error( + "Failed to set call widget `alwaysOnScreen` to false", + e, + ); + } + // On a normal user hangup we can shut down and close the widget. But if an + // error occurs we should keep the widget open until the user reads it. + if (reason === "user" && !getUrlParams().returnToLobby) { + try { + await widget.api.transport.send(ElementWidgetActions.Close, {}); + } catch (e) { + logger.error("Failed to send close action", e); + } + widget.api.transport.stop(); + } } - }) - .catch(() => - logger.error( - "could failed to play leave audio or send posthog leave event", - ), - ); + }); }, [ setJoined, @@ -367,24 +396,11 @@ export const GroupCallView: FC = ({ ); useEffect(() => { - if (widget && joined) { + if (widget && joined) // set widget to sticky once joined. widget.api.setAlwaysOnScreen(true).catch((e) => { logger.error("Error calling setAlwaysOnScreen(true)", e); }); - - const onHangup = (ev: CustomEvent): void => { - widget.api.transport.reply(ev.detail, {}); - // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts. - leaveRTCSession(rtcSession, "user").catch((e) => { - logger.error("Failed to leave RTC session", e); - }); - }; - widget.lazyActions.once(ElementWidgetActions.HangupCall, onHangup); - return (): void => { - widget.lazyActions.off(ElementWidgetActions.HangupCall, onHangup); - }; - } }, [widget, joined, rtcSession]); const joinRule = useJoinRule(room); diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 14f18fb7a..20d53b3a6 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -151,7 +151,7 @@ export const ActiveCall: FC = (props) => { ); setVm(vm); - const sub = vm.left$.subscribe(props.onLeft); + const sub = vm.leave$.subscribe(props.onLeft); return (): void => { vm.destroy(); sub.unsubscribe(); @@ -798,7 +798,7 @@ export const InCallView: FC = ({ (); - - public leave(): void { - this.leave$.next("user"); - } - - private readonly _left$ = new Subject< - "decline" | "timeout" | "user" | "allOthersLeft" - >(); - public left$ = this._left$.asObservable(); - private readonly connectionInstructions$ = this.join$.pipe( switchMap(() => this.remoteConnections$), startWith(new Map()), @@ -1154,7 +1142,7 @@ export class CallViewModel extends ViewModel { ); private readonly allOthersLeft$ = this.matrixUserChanges$.pipe( - map(({ userIds, leftUserIds }) => { + filter(({ userIds, leftUserIds }) => { if (!this.userId) { logger.warn("Could not access user ID to compute allOthersLeft"); return false; @@ -1163,12 +1151,40 @@ export class CallViewModel extends ViewModel { userIds.size === 1 && userIds.has(this.userId) && leftUserIds.size > 0 ); }), - startWith(false), + map(() => "allOthersLeft" as const), + ); + + // Public for testing + public readonly autoLeave$ = merge( + this.options.autoLeaveWhenOthersLeft ? this.allOthersLeft$ : NEVER, + this.callPickupState$.pipe( + filter((state) => state === "timeout" || state === "decline"), + ), ); - public readonly autoLeave$ = this.options.autoLeaveWhenOthersLeft - ? this.allOthersLeft$ - : NEVER; + private readonly userHangup$ = new Subject(); + public hangup(): void { + this.userHangup$.next(); + } + + private readonly widgetHangup$ = + widget === null + ? NEVER + : ( + fromEvent( + widget.lazyActions, + ElementWidgetActions.HangupCall, + ) as Observable<[CustomEvent]> + ).pipe(tap(([ev]) => widget!.api.transport.reply(ev.detail, {}))); + + public readonly leave$: Observable< + "user" | "timeout" | "decline" | "allOthersLeft" + > = merge( + this.autoLeave$, + merge(this.userHangup$, this.widgetHangup$).pipe( + map(() => "user" as const), + ), + ); /** * List of MediaItems that we want to display, that are of type ScreenShare @@ -1929,34 +1945,17 @@ export class CallViewModel extends ViewModel { ); }); - this.allOthersLeft$ - .pipe( - this.scope.bind(), - filter((l) => (l && this.options.autoLeaveWhenOthersLeft) ?? false), - distinctUntilChanged(), - ) - .subscribe(() => { - this.leave$.next("allOthersLeft"); - }); - - this.callPickupState$.pipe(this.scope.bind()).subscribe((state) => { - if (state === "timeout" || state === "decline") { - this.leave$.next(state); - } - }); - - this.leave$.pipe(this.scope.bind()).subscribe((reason) => { - const { confineToRoom } = this.urlParams; - leaveRTCSession(this.matrixRTCSession, "user") - // Only sends matrix leave event. The Livekit session will disconnect once the ActiveCall-view unmounts. - .then(() => { - if (!confineToRoom && !PosthogAnalytics.instance.isEnabled()) { - this._left$.next(reason); - } - }) - .catch((e) => { - logger.error("Error leaving RTC session", e); - }); + this.leave$.pipe(this.scope.bind()).subscribe(() => { + // Only sends Matrix leave event. The LiveKit session will disconnect once, uh... + // (TODO-MULTI-SFU does anything actually cause it to disconnect?) + void this.matrixRTCSession + .leaveRoomSession() + .catch((e) => logger.error("Error leaving RTC session", e)) + .then(async () => + widget?.api.transport + .send(ElementWidgetActions.HangupCall, {}) + .catch((e) => logger.error("Failed to send hangup action", e)), + ); }); // Pause upstream of all local media tracks when we're disconnected from From 530fbaf90afba3daf459d6df5e5264b9859a6409 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 24 Sep 2025 21:39:36 -0400 Subject: [PATCH 37/45] Clear up the room membership confusion around reading session members --- src/state/CallViewModel.ts | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 867e5e092..5f3650972 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -71,6 +71,7 @@ import { MembershipManagerEvent, Status, } from "matrix-js-sdk/lib/matrixrtc"; +import { type IWidgetApiRequest } from "matrix-widget-api"; import { ViewModel } from "./ViewModel"; import { @@ -127,7 +128,6 @@ import { type MuteStates } from "./MuteStates"; import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../widget"; -import { IWidgetApiRequest } from "matrix-widget-api"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -475,9 +475,11 @@ export class CallViewModel extends ViewModel { ), ); - // TODO-MULTI-SFU make sure that we consider the room memberships here as well (so that here we only have valid memberships) - // this also makes it possible to use this memberships$ list in all observables based on it. - // there should be no other call to: this.matrixRTCSession.memberships! + /** + * The MatrixRTC session participants. + */ + // Note that MatrixRTCSession already filters the call memberships by users + // that are joined to the room; we don't need to perform extra filtering here. public readonly memberships$ = this.scope.behavior( fromEvent( this.matrixRTCSession, @@ -735,19 +737,17 @@ export class CallViewModel extends ViewModel { // than on Chrome/Firefox). This means it is important that we multicast the result so that we // don't do this work more times than we need to. This is achieved by converting to a behavior: public readonly memberDisplaynames$ = this.scope.behavior( - merge( - // Handle call membership changes. - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ), - // Handle room membership changes (and displayname updates) - fromEvent(this.matrixRoom, RoomStateEvent.Members), - // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), - ).pipe( - startWith(null), - map(() => { - const memberships = this.matrixRTCSession.memberships; + combineLatest( + [ + // Handle call membership changes + this.memberships$, + // Additionally handle display name changes (implicitly reacting to them) + fromEvent(this.matrixRoom, RoomStateEvent.Members).pipe( + startWith(null), + ), + // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), + ], + (memberships, _displaynames) => { const displaynameMap = new Map([ ["local", this.matrixRoom.getMember(this.userId!)!.rawDisplayName], ]); @@ -771,7 +771,7 @@ export class CallViewModel extends ViewModel { ); } return displaynameMap; - }), + }, ), ); From 0759f9b27d1e277344605e674413be4cfb2901ef Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 25 Sep 2025 21:29:02 -0400 Subject: [PATCH 38/45] Don't render audio from participants that aren't meant to be publishing --- src/livekit/MatrixAudioRenderer.tsx | 47 +++++---- src/room/InCallView.tsx | 14 +-- src/state/CallViewModel.ts | 148 +++++++++++++++------------- src/state/Connection.ts | 9 +- 4 files changed, 121 insertions(+), 97 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 0b87b27b5..f402b32d7 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { getTrackReferenceId } from "@livekit/components-core"; -import { type Room as LivekitRoom } from "livekit-client"; +import { type Room as LivekitRoom, type Participant } from "livekit-client"; import { type RemoteAudioTrack, Track } from "livekit-client"; import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; import { @@ -14,7 +14,7 @@ import { AudioTrack, type AudioTrackProps, } from "@livekit/components-react"; -import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; +import { type RoomMember } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; @@ -22,13 +22,20 @@ import { useReactiveState } from "../useReactiveState"; import * as controls from "../controls"; import {} from "@livekit/components-core"; export interface MatrixAudioRendererProps { + /** + * The service URL of the LiveKit room. + */ + url: string; + livekitRoom: LivekitRoom; /** * The list of participants to render audio for. * This list needs to be composed based on the matrixRTC members so that we do not play audio from users * that are not expected to be in the rtc session. */ - members: CallMembership[]; - livekitRoom: LivekitRoom; + participants: { + participant: Participant; + member: RoomMember; + }[]; /** * If set to `true`, mutes all audio tracks rendered by the component. * @remarks @@ -52,14 +59,14 @@ export interface MatrixAudioRendererProps { * @public */ export function LivekitRoomAudioRenderer({ - members, - muted, + url, livekitRoom, + participants, + muted, }: MatrixAudioRendererProps): ReactNode { - const validIdentities = useMemo( - () => - new Set(members?.map((member) => `${member.sender}:${member.deviceId}`)), - [members], + const participantSet = useMemo( + () => new Set(participants.map(({ participant }) => participant)), + [participants], ); const loggedInvalidIdentities = useRef(new Set()); @@ -71,11 +78,11 @@ export function LivekitRoomAudioRenderer({ * @param identity The identity of the track that is invalid * @param validIdentities The list of valid identities */ - const logInvalid = (identity: string, validIdentities: Set): void => { + const logInvalid = (identity: string): void => { if (loggedInvalidIdentities.current.has(identity)) return; logger.warn( - `[MatrixAudioRenderer] Audio track ${identity} has no matching matrix call member`, - `current members: ${Array.from(validIdentities.values())}`, + `[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`, + `current members: ${participants.map((p) => p.participant.identity)}`, `track will not get rendered`, ); loggedInvalidIdentities.current.add(identity); @@ -93,23 +100,27 @@ export function LivekitRoomAudioRenderer({ room: livekitRoom, }, ).filter((ref) => { - const isValid = validIdentities?.has(ref.participant.identity); + const isValid = participantSet?.has(ref.participant); if (!isValid && !ref.participant.isLocal) - logInvalid(ref.participant.identity, validIdentities); + logInvalid(ref.participant.identity); return ( !ref.participant.isLocal && ref.publication.kind === Track.Kind.Audio && isValid ); }); + useEffect(() => { - if (!tracks.some((t) => !validIdentities.has(t.participant.identity))) { + if ( + loggedInvalidIdentities.current.size && + tracks.every((t) => participantSet.has(t.participant)) + ) { logger.debug( - `[MatrixAudioRenderer] All audio tracks have a matching matrix call member identity.`, + `[MatrixAudioRenderer] All audio tracks from ${url} have a matching matrix call member identity.`, ); loggedInvalidIdentities.current.clear(); } - }, [tracks, validIdentities]); + }, [tracks, participantSet, url]); // This component is also (in addition to the "only play audio for connected members" logic above) // responsible for mimicking earpiece audio on iPhones. diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 20d53b3a6..db2c0f2af 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -286,6 +286,8 @@ export const InCallView: FC = ({ () => void toggleRaisedHand(), ); + const allLivekitRooms = useBehavior(vm.allLivekitRooms$); + const participantsByRoom = useBehavior(vm.participantsByRoom$); const participantCount = useBehavior(vm.participantCount$); const reconnecting = useBehavior(vm.reconnecting$); const windowMode = useBehavior(vm.windowMode$); @@ -739,9 +741,6 @@ export const InCallView: FC = ({ matrixRoom.roomId, ); - const allLivekitRooms = useBehavior(vm.allLivekitRooms$); - const memberships = useBehavior(vm.memberships$); - const buttons: JSX.Element[] = []; buttons.push( @@ -862,11 +861,12 @@ export const InCallView: FC = ({ ) } - {allLivekitRooms.map((roomItem) => ( + {participantsByRoom.map(({ livekitRoom, url, participants }) => ( ))} diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 3b1cdc961..3dff08d30 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -480,7 +480,7 @@ export class CallViewModel extends ViewModel { */ // Note that MatrixRTCSession already filters the call memberships by users // that are joined to the room; we don't need to perform extra filtering here. - public readonly memberships$ = this.scope.behavior( + private readonly memberships$ = this.scope.behavior( fromEvent( this.matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged, @@ -679,16 +679,19 @@ export class CallViewModel extends ViewModel { // in a split-brained state. private readonly pretendToBeDisconnected$ = this.reconnecting$; - private readonly participants$ = this.scope.behavior< + public readonly participantsByRoom$ = this.scope.behavior< { - participant: LocalParticipant | RemoteParticipant; - member: RoomMember; livekitRoom: LivekitRoom; + url: string; + participants: { + participant: LocalParticipant | RemoteParticipant; + member: RoomMember; + }[]; }[] >( - from(this.localConnection) + combineLatest([this.localConnection, this.localFocus]) .pipe( - switchMap((localConnection) => { + switchMap(([localConnection, localFocus]) => { const memberError = (): never => { throw new Error("No room member for call membership"); }; @@ -696,32 +699,41 @@ export class CallViewModel extends ViewModel { participant: localConnection.livekitRoom.localParticipant, member: this.matrixRoom.getMember(this.userId ?? "") ?? memberError(), - livekitRoom: localConnection.livekitRoom, }; + return this.remoteConnections$.pipe( switchMap((connections) => combineLatest( - [localConnection, ...connections.values()].map((c) => + [ + [localFocus.livekit_service_url, localConnection] as const, + ...connections, + ].map(([url, c]) => c.publishingParticipants$.pipe( - map((ps) => - ps.map(({ participant, membership }) => ({ + map((ps) => { + const participants: { + participant: LocalParticipant | RemoteParticipant; + member: RoomMember; + }[] = ps.map(({ participant, membership }) => ({ participant, member: getRoomMemberFromRtcMember( membership, this.matrixRoom, )?.member ?? memberError(), + })); + if (c === localConnection) + participants.push(localParticipant); + + return { livekitRoom: c.livekitRoom, - })), - ), + url, + participants, + }; + }), ), ), ), ), - map((remoteParticipants) => [ - localParticipant, - ...remoteParticipants.flat(1), - ]), ); }), ) @@ -798,7 +810,7 @@ export class CallViewModel extends ViewModel { */ private readonly mediaItems$ = this.scope.behavior( combineLatest([ - this.participants$, + this.participantsByRoom$, duplicateTiles.value$, this.memberships$, showNonMemberTiles.value$, @@ -806,71 +818,75 @@ export class CallViewModel extends ViewModel { scan( ( prevItems, - [participants, duplicateTiles, memberships, showNonMemberTiles], + [participantsByRoom, duplicateTiles, memberships, showNonMemberTiles], ) => { const newItems: Map = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (const { participant, member, livekitRoom } of participants) { - const matrixId = participant.isLocal - ? "local" - : participant.identity; - for (let i = 0; i < 1 + duplicateTiles; i++) { - const mediaId = `${matrixId}:${i}`; - let prevMedia = prevItems.get(mediaId); - if (prevMedia && prevMedia instanceof UserMedia) { - prevMedia.updateParticipant(participant); - if (prevMedia.vm.member === undefined) { - // We have a previous media created because of the `debugShowNonMember` flag. - // In this case we actually replace the media item. - // This "hack" never occurs if we do not use the `debugShowNonMember` debugging - // option and if we always find a room member for each rtc member (which also - // only fails if we have a fundamental problem) - prevMedia = undefined; + for (const { livekitRoom, participants } of participantsByRoom) { + for (const { participant, member } of participants) { + const matrixId = participant.isLocal + ? "local" + : participant.identity; + + for (let i = 0; i < 1 + duplicateTiles; i++) { + const mediaId = `${matrixId}:${i}`; + let prevMedia = prevItems.get(mediaId); + if (prevMedia && prevMedia instanceof UserMedia) { + prevMedia.updateParticipant(participant); + if (prevMedia.vm.member === undefined) { + // We have a previous media created because of the `debugShowNonMember` flag. + // In this case we actually replace the media item. + // This "hack" never occurs if we do not use the `debugShowNonMember` debugging + // option and if we always find a room member for each rtc member (which also + // only fails if we have a fundamental problem) + prevMedia = undefined; + } } - } - yield [ - mediaId, - // We create UserMedia with or without a participant. - // This will be the initial value of a BehaviourSubject. - // Once a participant appears we will update the BehaviourSubject. (see above) - prevMedia ?? - new UserMedia( - mediaId, - member, - participant, - this.options.encryptionSystem, - livekitRoom, - this.mediaDevices, - this.pretendToBeDisconnected$, - this.memberDisplaynames$.pipe( - map((m) => m.get(matrixId) ?? "[👻]"), - ), - this.handsRaised$.pipe( - map((v) => v[matrixId]?.time ?? null), - ), - this.reactions$.pipe( - map((v) => v[matrixId] ?? undefined), - ), - ), - ]; - if (participant?.isScreenShareEnabled) { - const screenShareId = `${mediaId}:screen-share`; yield [ - screenShareId, - prevItems.get(screenShareId) ?? - new ScreenShare( - screenShareId, + mediaId, + // We create UserMedia with or without a participant. + // This will be the initial value of a BehaviourSubject. + // Once a participant appears we will update the BehaviourSubject. (see above) + prevMedia ?? + new UserMedia( + mediaId, member, participant, this.options.encryptionSystem, livekitRoom, + this.mediaDevices, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( map((m) => m.get(matrixId) ?? "[👻]"), ), + this.handsRaised$.pipe( + map((v) => v[matrixId]?.time ?? null), + ), + this.reactions$.pipe( + map((v) => v[matrixId] ?? undefined), + ), ), ]; + + if (participant?.isScreenShareEnabled) { + const screenShareId = `${mediaId}:screen-share`; + yield [ + screenShareId, + prevItems.get(screenShareId) ?? + new ScreenShare( + screenShareId, + member, + participant, + this.options.encryptionSystem, + livekitRoom, + this.pretendToBeDisconnected$, + this.memberDisplaynames$.pipe( + map((m) => m.get(matrixId) ?? "[👻]"), + ), + ), + ]; + } } } } diff --git a/src/state/Connection.ts b/src/state/Connection.ts index e14ddc9a9..2513382ce 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -93,11 +93,9 @@ export class Connection { ); this.publishingParticipants$ = this.scope.behavior( - combineLatest([ - this.participantsIncludingSubscribers$, - this.membershipsFocusMap$, - ]).pipe( - map(([participants, membershipsFocusMap]) => + combineLatest( + [this.participantsIncludingSubscribers$, this.membershipsFocusMap$], + (participants, membershipsFocusMap) => membershipsFocusMap // Find all members that claim to publish on this connection .flatMap(({ membership, focus }) => @@ -113,7 +111,6 @@ export class Connection { ); return participant ? [{ participant, membership }] : []; }), - ), ), [], ); From dbdf853d558dc0342cf68098cf09ced93ab18f9e Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 26 Sep 2025 13:20:55 -0400 Subject: [PATCH 39/45] Stop connections on view model destroy --- src/state/CallViewModel.ts | 1 - src/state/Connection.ts | 13 +++++++------ src/state/ObservableScope.ts | 16 ++++++++++++++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 3dff08d30..439d26626 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -359,7 +359,6 @@ class UserMedia { public destroy(): void { this.scope.end(); - this.vm.destroy(); } } diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 2513382ce..db456ba03 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -55,6 +55,7 @@ export class Connection { } public stop(): void { + if (this.stopped) return; void this.livekitRoom.disconnect(); this.stopped = true; } @@ -117,6 +118,8 @@ export class Connection { this.connectionState$ = this.scope.behavior( connectionStateObserver(this.livekitRoom), ); + + this.scope.onEnd(() => this.stop()); } } @@ -137,11 +140,6 @@ export class PublishConnection extends Connection { } } - public stop(): void { - void this.livekitRoom.disconnect(); - this.stopped = true; - } - public constructor( focus: LivekitFocus, livekitAlias: string, @@ -220,7 +218,10 @@ export class PublishConnection extends Connection { } return this.livekitRoom.localParticipant.isCameraEnabled; }); - // TODO-MULTI-SFU: Unset mute state handlers on destroy + this.scope.onEnd(() => { + this.muteStates.audio.unsetHandler(); + this.muteStates.video.unsetHandler(); + }); const syncDevice = ( kind: MediaDeviceKind, diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index 1cddfbff4..fe99d89bf 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -36,11 +36,16 @@ export class ObservableScope { return this.bindImpl; } - private readonly shareImpl: MonoTypeOperator = share({ resetOnError: false, resetOnComplete: false, resetOnRefCountZero: false }) + private readonly shareImpl: MonoTypeOperator = share({ + resetOnError: false, + resetOnComplete: false, + resetOnRefCountZero: false, + }); /** * Shares (multicasts) the Observable as a hot Observable. */ - public readonly share: MonoTypeOperator = (input$) => input$.pipe(this.bindImpl, this.shareImpl) + public readonly share: MonoTypeOperator = (input$) => + input$.pipe(this.bindImpl, this.shareImpl); /** * Converts an Observable to a Behavior. If no initial value is specified, the @@ -76,6 +81,13 @@ export class ObservableScope { this.ended$.next(); this.ended$.complete(); } + + /** + * Register a callback to be executed when the scope is ended. + */ + public onEnd(callback: () => void): void { + this.ended$.subscribe(callback); + } } /** From a4a0a58a72d1fe034fc0e53669ae375fc8102ee8 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 26 Sep 2025 13:26:42 -0400 Subject: [PATCH 40/45] Remove the option to show non-member ("ghost") participants As we'd like to get the multi-SFU feature branch shipped, this is not the most important debugging tool to expend effort on at the moment. --- locales/en/app.json | 1 - src/settings/DeveloperSettingsTab.tsx | 18 --- src/settings/settings.ts | 4 - src/state/CallViewModel.test.ts | 48 ------- src/state/CallViewModel.ts | 189 ++++++++------------------ 5 files changed, 60 insertions(+), 200 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index 007e372a0..dc027c922 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -74,7 +74,6 @@ "matrix_id": "Matrix ID: {{id}}", "mute_all_audio": "Mute all audio (participants, reactions, join sounds)", "show_connection_stats": "Show connection statistics", - "show_non_member_tiles": "Show tiles for non-member media", "url_params": "URL parameters", "use_new_membership_manager": "Use the new implementation of the call MembershipManager", "use_to_device_key_transport": "Use to device key transport. This will fallback to room key transport when another call member sent a room key" diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index d503385b5..1949ecf7a 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -13,7 +13,6 @@ import { useSetting, duplicateTiles as duplicateTilesSetting, debugTileLayout as debugTileLayoutSetting, - showNonMemberTiles as showNonMemberTilesSetting, showConnectionStats as showConnectionStatsSetting, useNewMembershipManager as useNewMembershipManagerSetting, useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting, @@ -35,9 +34,6 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { const [debugTileLayout, setDebugTileLayout] = useSetting( debugTileLayoutSetting, ); - const [showNonMemberTiles, setShowNonMemberTiles] = useSetting( - showNonMemberTilesSetting, - ); const [showConnectionStats, setShowConnectionStats] = useSetting( showConnectionStatsSetting, @@ -128,20 +124,6 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { } /> - - ): void => { - setShowNonMemberTiles(event.target.checked); - }, - [setShowNonMemberTiles], - )} - /> - ( - "show-non-member-tiles", - false, -); export const debugTileLayout = new Setting("debug-tile-layout", false); export const showConnectionStats = new Setting( diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index b736b780e..07c78ef66 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -69,7 +69,6 @@ import { } from "../livekit/useECConnectionState"; import { E2eeType } from "../e2ee/e2eeType"; import type { RaisedHandInfo } from "../reactions"; -import { showNonMemberTiles } from "../settings/settings"; import { alice, aliceDoppelganger, @@ -824,53 +823,6 @@ test("participants must have a MatrixRTCSession to be visible", () => { }); }); -test("shows participants without MatrixRTCSession when enabled in settings", () => { - try { - // enable the setting: - showNonMemberTiles.setValue(true); - withTestScheduler(({ behavior, expectObservable }) => { - const scenarioInputMarbles = " abc"; - const expectedLayoutMarbles = "abc"; - - withCallViewModel( - { - remoteParticipants$: behavior(scenarioInputMarbles, { - a: [], - b: [aliceParticipant], - c: [aliceParticipant, bobParticipant], - }), - rtcMembers$: constant([localRtcMember]), // No one else joins the MatrixRTC session - }, - (vm) => { - vm.setGridMode("grid"); - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: ["local:0"], - }, - b: { - type: "one-on-one", - local: "local:0", - remote: `${aliceId}:0`, - }, - c: { - type: "grid", - spotlight: undefined, - grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], - }, - }, - ); - }, - ); - }); - } finally { - showNonMemberTiles.setValue(showNonMemberTiles.defaultValue); - } -}); - it("should show at least one tile per MatrixRTCSession", () => { withTestScheduler(({ behavior, expectObservable }) => { // iterate through some combinations of MatrixRTC memberships diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 439d26626..7e3a5bdf8 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -92,7 +92,6 @@ import { duplicateTiles, playReactionsSound, showReactions, - showNonMemberTiles, } from "../settings/settings"; import { isFirefox } from "../Platform"; import { setPipEnabled$ } from "../controls"; @@ -812,152 +811,84 @@ export class CallViewModel extends ViewModel { this.participantsByRoom$, duplicateTiles.value$, this.memberships$, - showNonMemberTiles.value$, ]).pipe( - scan( - ( - prevItems, - [participantsByRoom, duplicateTiles, memberships, showNonMemberTiles], - ) => { - const newItems: Map = new Map( - function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (const { livekitRoom, participants } of participantsByRoom) { - for (const { participant, member } of participants) { - const matrixId = participant.isLocal - ? "local" - : participant.identity; - - for (let i = 0; i < 1 + duplicateTiles; i++) { - const mediaId = `${matrixId}:${i}`; - let prevMedia = prevItems.get(mediaId); - if (prevMedia && prevMedia instanceof UserMedia) { - prevMedia.updateParticipant(participant); - if (prevMedia.vm.member === undefined) { - // We have a previous media created because of the `debugShowNonMember` flag. - // In this case we actually replace the media item. - // This "hack" never occurs if we do not use the `debugShowNonMember` debugging - // option and if we always find a room member for each rtc member (which also - // only fails if we have a fundamental problem) - prevMedia = undefined; - } + scan((prevItems, [participantsByRoom, duplicateTiles, memberships]) => { + const newItems: Map = new Map( + function* (this: CallViewModel): Iterable<[string, MediaItem]> { + for (const { livekitRoom, participants } of participantsByRoom) { + for (const { participant, member } of participants) { + const matrixId = participant.isLocal + ? "local" + : participant.identity; + + for (let i = 0; i < 1 + duplicateTiles; i++) { + const mediaId = `${matrixId}:${i}`; + let prevMedia = prevItems.get(mediaId); + if (prevMedia && prevMedia instanceof UserMedia) { + prevMedia.updateParticipant(participant); + if (prevMedia.vm.member === undefined) { + // We have a previous media created because of the `debugShowNonMember` flag. + // In this case we actually replace the media item. + // This "hack" never occurs if we do not use the `debugShowNonMember` debugging + // option and if we always find a room member for each rtc member (which also + // only fails if we have a fundamental problem) + prevMedia = undefined; } + } + yield [ + mediaId, + // We create UserMedia with or without a participant. + // This will be the initial value of a BehaviourSubject. + // Once a participant appears we will update the BehaviourSubject. (see above) + prevMedia ?? + new UserMedia( + mediaId, + member, + participant, + this.options.encryptionSystem, + livekitRoom, + this.mediaDevices, + this.pretendToBeDisconnected$, + this.memberDisplaynames$.pipe( + map((m) => m.get(matrixId) ?? "[👻]"), + ), + this.handsRaised$.pipe( + map((v) => v[matrixId]?.time ?? null), + ), + this.reactions$.pipe( + map((v) => v[matrixId] ?? undefined), + ), + ), + ]; + + if (participant?.isScreenShareEnabled) { + const screenShareId = `${mediaId}:screen-share`; yield [ - mediaId, - // We create UserMedia with or without a participant. - // This will be the initial value of a BehaviourSubject. - // Once a participant appears we will update the BehaviourSubject. (see above) - prevMedia ?? - new UserMedia( - mediaId, + screenShareId, + prevItems.get(screenShareId) ?? + new ScreenShare( + screenShareId, member, participant, this.options.encryptionSystem, livekitRoom, - this.mediaDevices, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( map((m) => m.get(matrixId) ?? "[👻]"), ), - this.handsRaised$.pipe( - map((v) => v[matrixId]?.time ?? null), - ), - this.reactions$.pipe( - map((v) => v[matrixId] ?? undefined), - ), ), ]; - - if (participant?.isScreenShareEnabled) { - const screenShareId = `${mediaId}:screen-share`; - yield [ - screenShareId, - prevItems.get(screenShareId) ?? - new ScreenShare( - screenShareId, - member, - participant, - this.options.encryptionSystem, - livekitRoom, - this.pretendToBeDisconnected$, - this.memberDisplaynames$.pipe( - map((m) => m.get(matrixId) ?? "[👻]"), - ), - ), - ]; - } } } } - }.bind(this)(), - ); + } + }.bind(this)(), + ); - // Generate non member items (items without a corresponding MatrixRTC member) - // Those items should not be rendered, they are participants in LiveKit that do not have a corresponding - // MatrixRTC members. This cannot be any good: - // - A malicious user impersonates someone - // - Someone injects abusive content - // - The user cannot have encryption keys so it makes no sense to participate - // We can only trust users that have a MatrixRTC member event. - // - // This is still available as a debug option. This can be useful - // - If one wants to test scalability using the LiveKit CLI. - // - If an experimental project does not yet do the MatrixRTC bits. - // - If someone wants to debug if the LiveKit connection works but MatrixRTC room state failed to arrive. - // TODO-MULTI-SFU - // const newNonMemberItems = showNonMemberTiles - // ? new Map( - // function* ( - // this: CallViewModel, - // ): Iterable<[string, MediaItem]> { - // for (const participant of remoteParticipants) { - // for (let i = 0; i < 1 + duplicateTiles; i++) { - // const maybeNonMemberParticipantId = - // participant.identity + ":" + i; - // if (!newItems.has(maybeNonMemberParticipantId)) { - // const nonMemberId = maybeNonMemberParticipantId; - // yield [ - // nonMemberId, - // prevItems.get(nonMemberId) ?? - // new UserMedia( - // nonMemberId, - // undefined, - // participant, - // this.options.encryptionSystem, - // localConnection.livekitRoom, - // this.mediaDevices, - // this.pretendToBeDisconnected$, - // this.memberDisplaynames$.pipe( - // map( - // (m) => - // m.get(participant.identity) ?? "[👻]", - // ), - // ), - // of(null), - // of(null), - // ), - // ]; - // } - // } - // } - // }.bind(this)(), - // ) - // : new Map(); - // if (newNonMemberItems.size > 0) { - // logger.debug("Added NonMember items: ", newNonMemberItems); - // } - - const combinedNew = new Map([ - // ...newNonMemberItems.entries(), - ...newItems.entries(), - ]); - - for (const [id, t] of prevItems) - if (!combinedNew.has(id)) t.destroy(); - return combinedNew; - }, - new Map(), - ), + for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy(); + return newItems; + }, new Map()), map((mediaItems) => [...mediaItems.values()]), finalizeValue((ts) => { for (const t of ts) t.destroy(); From 2819c7959013a6a287ebc9960f898a4b454cb4a1 Mon Sep 17 00:00:00 2001 From: Timo K Date: Tue, 30 Sep 2025 16:47:45 +0200 Subject: [PATCH 41/45] use updated multi sfu js-sdk Signed-off-by: Timo K --- src/room/useActiveFocus.ts | 45 ----------------------- src/rtcSessionHelpers.ts | 75 +++++++++++++++++++------------------- src/state/CallViewModel.ts | 6 ++- src/state/Connection.ts | 10 ++--- yarn.lock | 4 +- 5 files changed, 48 insertions(+), 92 deletions(-) delete mode 100644 src/room/useActiveFocus.ts diff --git a/src/room/useActiveFocus.ts b/src/room/useActiveFocus.ts deleted file mode 100644 index 7a8f45213..000000000 --- a/src/room/useActiveFocus.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type MatrixRTCSession, - MatrixRTCSessionEvent, -} from "matrix-js-sdk/lib/matrixrtc"; -import { useCallback, useRef } from "react"; -import { deepCompare } from "matrix-js-sdk/lib/utils"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { type LivekitFocus, isLivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; - -import { useTypedEventEmitterState } from "../useEvents"; - -/** - * Gets the currently active (livekit) focus for a MatrixRTC session - * This logic is specific to livekit foci where the whole call must use one - * and the same focus. - */ -export function useActiveLivekitFocus( - rtcSession: MatrixRTCSession, -): LivekitFocus | undefined { - const prevActiveFocus = useRef(undefined); - return useTypedEventEmitterState( - rtcSession, - MatrixRTCSessionEvent.MembershipsChanged, - useCallback(() => { - const f = rtcSession.getActiveFocus(); - // Only handle foci with type="livekit" for now. - if (f && isLivekitFocus(f) && !deepCompare(f, prevActiveFocus.current)) { - const oldestMembership = rtcSession.getOldestMembership(); - logger.info( - `Got new active focus from membership: ${oldestMembership?.sender}/${oldestMembership?.deviceId}. - Updated focus (focus switch) from ${JSON.stringify(prevActiveFocus.current)} to ${JSON.stringify(f)}`, - ); - prevActiveFocus.current = f; - } - return prevActiveFocus.current; - }, [rtcSession]), - ); -} diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index b6918f3ab..175b35f4c 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -6,11 +6,10 @@ Please see LICENSE in the repository root for full details. */ import { - isLivekitFocusConfig, - type LivekitFocusConfig, - type LivekitFocus, - type LivekitFocusSelection, type MatrixRTCSession, + isLivekitTransportConfig, + type LivekitTransportConfig, + type LivekitTransport, } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; @@ -24,13 +23,6 @@ import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; -export function makeActiveFocus(): LivekitFocusSelection { - return { - type: "livekit", - focus_selection: "oldest_membership", - }; -} - export function getLivekitAlias(rtcSession: MatrixRTCSession): string { // For now we assume everything is a room-scoped call return rtcSession.room.roomId; @@ -38,13 +30,13 @@ export function getLivekitAlias(rtcSession: MatrixRTCSession): string { async function makeFocusInternal( rtcSession: MatrixRTCSession, -): Promise { +): Promise { logger.log("Searching for a preferred focus"); const livekitAlias = getLivekitAlias(rtcSession); const urlFromStorage = localStorage.getItem("robin-matrixrtc-auth"); if (urlFromStorage !== null) { - const focusFromStorage: LivekitFocus = { + const focusFromStorage: LivekitTransport = { type: "livekit", livekit_service_url: urlFromStorage, livekit_alias: livekitAlias, @@ -57,7 +49,7 @@ async function makeFocusInternal( const domain = rtcSession.room.client.getDomain(); if (localStorage.getItem("timo-focus-url")) { const timoFocusUrl = localStorage.getItem("timo-focus-url")!; - const focusFromUrl: LivekitFocus = { + const focusFromUrl: LivekitTransport = { type: "livekit", livekit_service_url: timoFocusUrl, livekit_alias: livekitAlias, @@ -72,8 +64,8 @@ async function makeFocusInternal( FOCI_WK_KEY ]; if (Array.isArray(wellKnownFoci)) { - const focus: LivekitFocusConfig | undefined = wellKnownFoci.find( - (f) => f && isLivekitFocusConfig(f), + const focus: LivekitTransportConfig | undefined = wellKnownFoci.find( + (f) => f && isLivekitTransportConfig(f), ); if (focus !== undefined) { logger.log("Using LiveKit focus from .well-known: ", focus); @@ -84,7 +76,7 @@ async function makeFocusInternal( const urlFromConf = Config.get().livekit?.livekit_service_url; if (urlFromConf) { - const focusFromConf: LivekitFocus = { + const focusFromConf: LivekitTransport = { type: "livekit", livekit_service_url: urlFromConf, livekit_alias: livekitAlias, @@ -98,7 +90,7 @@ async function makeFocusInternal( export async function makeFocus( rtcSession: MatrixRTCSession, -): Promise { +): Promise { const focus = await makeFocusInternal(rtcSession); // this will call the jwt/sfu/get endpoint to pre create the livekit room. await getSFUConfigWithOpenID( @@ -111,10 +103,11 @@ export async function makeFocus( export async function enterRTCSession( rtcSession: MatrixRTCSession, - focus: LivekitFocus, + focus: LivekitTransport, encryptMedia: boolean, useNewMembershipManager = true, useExperimentalToDeviceTransport = false, + useMultiSfu = true, ): Promise { PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); @@ -127,25 +120,31 @@ export async function enterRTCSession( const useDeviceSessionMemberEvents = features?.feature_use_device_session_member_events; const { sendNotificationType: notificationType, callIntent } = getUrlParams(); - rtcSession.joinRoomSession([focus], focus, { - notificationType, - callIntent, - useNewMembershipManager, - manageMediaKeys: encryptMedia, - ...(useDeviceSessionMemberEvents !== undefined && { - useLegacyMemberEvents: !useDeviceSessionMemberEvents, - }), - delayedLeaveEventRestartMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_ms, - delayedLeaveEventDelayMs: - matrixRtcSessionConfig?.delayed_leave_event_delay_ms, - delayedLeaveEventRestartLocalTimeoutMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, - networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, - makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, - membershipEventExpiryMs: matrixRtcSessionConfig?.membership_event_expiry_ms, - useExperimentalToDeviceTransport, - }); + // Multi-sfu does not need a focus preferred list. just the focus that is actually used. + rtcSession.joinRoomSession( + useMultiSfu ? [focus] : [], + useMultiSfu ? focus : undefined, + { + notificationType, + callIntent, + useNewMembershipManager, + manageMediaKeys: encryptMedia, + ...(useDeviceSessionMemberEvents !== undefined && { + useLegacyMemberEvents: !useDeviceSessionMemberEvents, + }), + delayedLeaveEventRestartMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_ms, + delayedLeaveEventDelayMs: + matrixRtcSessionConfig?.delayed_leave_event_delay_ms, + delayedLeaveEventRestartLocalTimeoutMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, + networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, + makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, + membershipEventExpiryMs: + matrixRtcSessionConfig?.membership_event_expiry_ms, + useExperimentalToDeviceTransport, + }, + ); if (widget) { try { await widget.api.transport.send(ElementWidgetActions.JoinCall, {}); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 7e3a5bdf8..2f4bfa0ca 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -64,7 +64,7 @@ import { import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, - isLivekitFocus, + isLivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -493,7 +493,9 @@ export class CallViewModel extends ViewModel { map((memberships) => memberships.flatMap((m) => { const f = this.matrixRTCSession.resolveActiveFocus(m); - return f && isLivekitFocus(f) ? [{ membership: m, focus: f }] : []; + return f && isLivekitTransport(f) + ? [{ membership: m, focus: f }] + : []; }), ), ), diff --git a/src/state/Connection.ts b/src/state/Connection.ts index db456ba03..8eaed4635 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -18,7 +18,7 @@ import { } from "livekit-client"; import { type MatrixClient } from "matrix-js-sdk"; import { - type LivekitFocus, + type LivekitTransport, type CallMembership, } from "matrix-js-sdk/lib/matrixrtc"; import { @@ -72,12 +72,12 @@ export class Connection { public connectionState$: Behavior; public constructor( - protected readonly focus: LivekitFocus, + protected readonly focus: LivekitTransport, protected readonly livekitAlias: string, protected readonly client: MatrixClient, protected readonly scope: ObservableScope, protected readonly membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitFocus }[] + { membership: CallMembership; focus: LivekitTransport }[] >, e2eeLivekitOptions: E2EEOptions | undefined, livekitRoom: LivekitRoom | undefined = undefined, @@ -141,12 +141,12 @@ export class PublishConnection extends Connection { } public constructor( - focus: LivekitFocus, + focus: LivekitTransport, livekitAlias: string, client: MatrixClient, scope: ObservableScope, membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitFocus }[] + { membership: CallMembership; focus: LivekitTransport }[] >, devices: MediaDevices, private readonly muteStates: MuteStates, diff --git a/yarn.lock b/yarn.lock index 4429b7d49..a149eaf5c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10299,7 +10299,7 @@ __metadata: "matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU": version: 38.3.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=ca4a9c655537702daf9a69ed5d94831cebc49666" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=d94d02d19b9f17c724b5919b185fea3413dbf7a2" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0" @@ -10315,7 +10315,7 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/1fb0933d0bb686b0f290b1a62f75eec290b7c52a410d5968c2ccfb527a64e78a58012e1bd8f90c874d385dace3228b9a8c80e114ee227fc8a60e7c9611112ceb + checksum: 10c0/dc43617a9398754275e2025af7d5fdee1f2e01b89241fc7881c1206d925e83ad6fe55f439501ae34e734cfbfa5479f6bee3167f4828c913f4f33817d95850189 languageName: node linkType: hard From 68aae4a8e3e2cd208db5ccc402b134b34159af5c Mon Sep 17 00:00:00 2001 From: Timo K Date: Thu, 2 Oct 2025 11:23:11 +0200 Subject: [PATCH 42/45] fix another rename + another js-sdk bump Signed-off-by: Timo K --- src/utils/test.ts | 10 +++++++--- yarn.lock | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/utils/test.ts b/src/utils/test.ts index 842ca008c..519fdd509 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -16,12 +16,13 @@ import { } from "matrix-js-sdk"; import { CallMembership, - type Focus, + type Transport, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, MembershipManagerEvent, type SessionMembershipData, Status, + type LivekitFocusSelection, } from "matrix-js-sdk/lib/matrixrtc"; import { type MembershipManagerEventHandlerMap } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; import { @@ -172,8 +173,11 @@ export function mockRtcMembership( user: string | RoomMember, deviceId: string, callId = "", - fociPreferred: Focus[] = [], - focusActive: Focus = { type: "oldest_membership" }, + fociPreferred: Transport[] = [], + focusActive: LivekitFocusSelection = { + type: "livekit", + focus_selection: "oldest_membership", + }, membership: Partial = {}, ): CallMembership { const data: SessionMembershipData = { diff --git a/yarn.lock b/yarn.lock index a149eaf5c..197cee3e6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10299,7 +10299,7 @@ __metadata: "matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU": version: 38.3.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=d94d02d19b9f17c724b5919b185fea3413dbf7a2" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=a343e8c92a5a37f419eb1b762db3a123e41ef66d" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0" @@ -10315,7 +10315,7 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/dc43617a9398754275e2025af7d5fdee1f2e01b89241fc7881c1206d925e83ad6fe55f439501ae34e734cfbfa5479f6bee3167f4828c913f4f33817d95850189 + checksum: 10c0/4893878f2fe07b06334bab4674a01569037d0f3e737fef3f0bb97a98b01d71fc304627921673f128821a17d824de9b63cc06456db15f9d45eb10bba1ceacd5c5 languageName: node linkType: hard From 86fb026be86d7ea0fa904babd920b6e1d0b9f1c0 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 3 Oct 2025 14:43:22 -0400 Subject: [PATCH 43/45] Turn multi-SFU media transport into a developer option --- locales/en/app.json | 3 +- src/room/GroupCallErrorBoundary.test.tsx | 8 +- src/room/GroupCallView.test.tsx | 4 +- src/rtcSessionHelpers.ts | 64 ++- src/settings/DeveloperSettingsTab.tsx | 16 + src/settings/settings.ts | 2 + src/state/Async.ts | 44 ++ src/state/CallViewModel.ts | 526 ++++++++++++++--------- src/state/Connection.ts | 57 +-- src/state/MuteStates.ts | 2 +- src/state/ObservableScope.ts | 19 +- src/utils/errors.ts | 10 +- 12 files changed, 463 insertions(+), 292 deletions(-) create mode 100644 src/state/Async.ts diff --git a/locales/en/app.json b/locales/en/app.json index dc027c922..704f68ac0 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -72,6 +72,7 @@ "livekit_server_info": "LiveKit Server Info", "livekit_sfu": "LiveKit SFU: {{url}}", "matrix_id": "Matrix ID: {{id}}", + "multi_sfu": "Multi-SFU media transport", "mute_all_audio": "Mute all audio (participants, reactions, join sounds)", "show_connection_stats": "Show connection statistics", "url_params": "URL parameters", @@ -91,7 +92,7 @@ "generic_description": "Submitting debug logs will help us track down the problem.", "insufficient_capacity": "Insufficient capacity", "insufficient_capacity_description": "The server has reached its maximum capacity and you cannot join the call at this time. Try again later, or contact your server admin if the problem persists.", - "matrix_rtc_focus_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).", + "matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).", "open_elsewhere": "Opened in another tab", "open_elsewhere_description": "{{brand}} has been opened in another tab. If that doesn't sound right, try reloading the page.", "room_creation_restricted": "Failed to create call", diff --git a/src/room/GroupCallErrorBoundary.test.tsx b/src/room/GroupCallErrorBoundary.test.tsx index 519129566..223389249 100644 --- a/src/room/GroupCallErrorBoundary.test.tsx +++ b/src/room/GroupCallErrorBoundary.test.tsx @@ -26,7 +26,7 @@ import { E2EENotSupportedError, type ElementCallError, InsufficientCapacityError, - MatrixRTCFocusMissingError, + MatrixRTCTransportMissingError, UnknownCallError, } from "../utils/errors.ts"; import { mockConfig } from "../utils/test.ts"; @@ -34,7 +34,7 @@ import { ElementWidgetActions, type WidgetHelpers } from "../widget.ts"; test.each([ { - error: new MatrixRTCFocusMissingError("example.com"), + error: new MatrixRTCTransportMissingError("example.com"), expectedTitle: "Call is not supported", }, { @@ -85,7 +85,7 @@ test.each([ ); test("should render the error page with link back to home", async () => { - const error = new MatrixRTCFocusMissingError("example.com"); + const error = new MatrixRTCTransportMissingError("example.com"); const TestComponent = (): ReactNode => { throw error; }; @@ -213,7 +213,7 @@ describe("Rageshake button", () => { }); test("should have a close button in widget mode", async () => { - const error = new MatrixRTCFocusMissingError("example.com"); + const error = new MatrixRTCTransportMissingError("example.com"); const TestComponent = (): ReactNode => { throw error; }; diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx index bf5d1fefc..b8bc2f534 100644 --- a/src/room/GroupCallView.test.tsx +++ b/src/room/GroupCallView.test.tsx @@ -42,7 +42,7 @@ import { import { GroupCallView } from "./GroupCallView"; import { type WidgetHelpers } from "../widget"; import { LazyEventEmitter } from "../LazyEventEmitter"; -import { MatrixRTCFocusMissingError } from "../utils/errors"; +import { MatrixRTCTransportMissingError } from "../utils/errors"; import { ProcessorProvider } from "../livekit/TrackProcessorContext"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { HeaderStyle } from "../UrlParams"; @@ -258,7 +258,7 @@ test("GroupCallView leaves the session when an error occurs", async () => { test("GroupCallView shows errors that occur during joining", async () => { const user = userEvent.setup(); - enterRTCSession.mockRejectedValue(new MatrixRTCFocusMissingError("")); + enterRTCSession.mockRejectedValue(new MatrixRTCTransportMissingError("")); onTestFinished(() => { enterRTCSession.mockReset(); }); diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 175b35f4c..3cdd82e71 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -17,7 +17,7 @@ import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import { PosthogAnalytics } from "./analytics/PosthogAnalytics"; import { Config } from "./config/Config"; import { ElementWidgetActions, widget, type WidgetHelpers } from "./widget"; -import { MatrixRTCFocusMissingError } from "./utils/errors"; +import { MatrixRTCTransportMissingError } from "./utils/errors"; import { getUrlParams } from "./UrlParams"; import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; @@ -28,35 +28,31 @@ export function getLivekitAlias(rtcSession: MatrixRTCSession): string { return rtcSession.room.roomId; } -async function makeFocusInternal( +async function makeTransportInternal( rtcSession: MatrixRTCSession, ): Promise { - logger.log("Searching for a preferred focus"); + logger.log("Searching for a preferred transport"); const livekitAlias = getLivekitAlias(rtcSession); - const urlFromStorage = localStorage.getItem("robin-matrixrtc-auth"); + // TODO-MULTI-SFU: Either remove this dev tool or make it more official + const urlFromStorage = + localStorage.getItem("robin-matrixrtc-auth") ?? + localStorage.getItem("timo-focus-url"); if (urlFromStorage !== null) { - const focusFromStorage: LivekitTransport = { + const transportFromStorage: LivekitTransport = { type: "livekit", livekit_service_url: urlFromStorage, livekit_alias: livekitAlias, }; - logger.log("Using LiveKit focus from local storage: ", focusFromStorage); - return focusFromStorage; + logger.log( + "Using LiveKit transport from local storage: ", + transportFromStorage, + ); + return transportFromStorage; } // Prioritize the .well-known/matrix/client, if available, over the configured SFU const domain = rtcSession.room.client.getDomain(); - if (localStorage.getItem("timo-focus-url")) { - const timoFocusUrl = localStorage.getItem("timo-focus-url")!; - const focusFromUrl: LivekitTransport = { - type: "livekit", - livekit_service_url: timoFocusUrl, - livekit_alias: livekitAlias, - }; - logger.log("Using LiveKit focus from localStorage: ", timoFocusUrl); - return focusFromUrl; - } if (domain) { // we use AutoDiscovery instead of relying on the MatrixClient having already // been fully configured and started @@ -64,46 +60,46 @@ async function makeFocusInternal( FOCI_WK_KEY ]; if (Array.isArray(wellKnownFoci)) { - const focus: LivekitTransportConfig | undefined = wellKnownFoci.find( + const transport: LivekitTransportConfig | undefined = wellKnownFoci.find( (f) => f && isLivekitTransportConfig(f), ); - if (focus !== undefined) { - logger.log("Using LiveKit focus from .well-known: ", focus); - return { ...focus, livekit_alias: livekitAlias }; + if (transport !== undefined) { + logger.log("Using LiveKit transport from .well-known: ", transport); + return { ...transport, livekit_alias: livekitAlias }; } } } const urlFromConf = Config.get().livekit?.livekit_service_url; if (urlFromConf) { - const focusFromConf: LivekitTransport = { + const transportFromConf: LivekitTransport = { type: "livekit", livekit_service_url: urlFromConf, livekit_alias: livekitAlias, }; - logger.log("Using LiveKit focus from config: ", focusFromConf); - return focusFromConf; + logger.log("Using LiveKit transport from config: ", transportFromConf); + return transportFromConf; } - throw new MatrixRTCFocusMissingError(domain ?? ""); + throw new MatrixRTCTransportMissingError(domain ?? ""); } -export async function makeFocus( +export async function makeTransport( rtcSession: MatrixRTCSession, ): Promise { - const focus = await makeFocusInternal(rtcSession); + const transport = await makeTransportInternal(rtcSession); // this will call the jwt/sfu/get endpoint to pre create the livekit room. await getSFUConfigWithOpenID( rtcSession.room.client, - focus.livekit_service_url, - focus.livekit_alias, + transport.livekit_service_url, + transport.livekit_alias, ); - return focus; + return transport; } export async function enterRTCSession( rtcSession: MatrixRTCSession, - focus: LivekitTransport, + transport: LivekitTransport, encryptMedia: boolean, useNewMembershipManager = true, useExperimentalToDeviceTransport = false, @@ -120,10 +116,10 @@ export async function enterRTCSession( const useDeviceSessionMemberEvents = features?.feature_use_device_session_member_events; const { sendNotificationType: notificationType, callIntent } = getUrlParams(); - // Multi-sfu does not need a focus preferred list. just the focus that is actually used. + // Multi-sfu does not need a preferred foci list. just the focus that is actually used. rtcSession.joinRoomSession( - useMultiSfu ? [focus] : [], - useMultiSfu ? focus : undefined, + useMultiSfu ? [] : [transport], + useMultiSfu ? transport : undefined, { notificationType, callIntent, diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 1949ecf7a..36c8a2e6c 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -16,6 +16,7 @@ import { showConnectionStats as showConnectionStatsSetting, useNewMembershipManager as useNewMembershipManagerSetting, useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting, + multiSfu as multiSfuSetting, muteAllAudio as muteAllAudioSetting, alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, } from "./settings"; @@ -50,6 +51,7 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { useExperimentalToDeviceTransport, setUseExperimentalToDeviceTransport, ] = useSetting(useExperimentalToDeviceTransportSetting); + const [multiSfu, setMultiSfu] = useSetting(multiSfuSetting); const [muteAllAudio, setMuteAllAudio] = useSetting(muteAllAudioSetting); @@ -166,6 +168,20 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { )} /> + + ): void => { + setMultiSfu(event.target.checked); + }, + [setMultiSfu], + )} + /> + ( true, ); +export const multiSfu = new Setting("multi-sfu", false); + export const muteAllAudio = new Setting("mute-all-audio", false); export const alwaysShowSelf = new Setting("always-show-self", true); diff --git a/src/state/Async.ts b/src/state/Async.ts new file mode 100644 index 000000000..2baa674c1 --- /dev/null +++ b/src/state/Async.ts @@ -0,0 +1,44 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + catchError, + from, + map, + Observable, + of, + startWith, + switchMap, +} from "rxjs"; + +export type Async = + | { state: "loading" } + | { state: "error"; value: Error } + | { state: "ready"; value: A }; + +export const loading: Async = { state: "loading" }; +export function error(value: Error): Async { + return { state: "error", value }; +} +export function ready(value: A): Async { + return { state: "ready", value }; +} + +export function async(promise: Promise): Observable> { + return from(promise).pipe( + map(ready), + startWith(loading), + catchError((e) => of(error(e))), + ); +} + +export function mapAsync( + async: Async, + project: (value: A) => B, +): Async { + return async.state === "ready" ? ready(project(async.value)) : async; +} diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 2f4bfa0ca..8988e5180 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -28,6 +28,7 @@ import { EventType, RoomEvent, } from "matrix-js-sdk"; +import { deepCompare } from "matrix-js-sdk/lib/utils"; import { BehaviorSubject, EMPTY, @@ -48,6 +49,7 @@ import { of, pairwise, race, + repeat, scan, skip, skipWhile, @@ -57,6 +59,7 @@ import { switchScan, take, takeUntil, + takeWhile, tap, throttleTime, timer, @@ -65,6 +68,7 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembership, isLivekitTransport, + type LivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -90,6 +94,7 @@ import { import { ObservableScope } from "./ObservableScope"; import { duplicateTiles, + multiSfu, playReactionsSound, showReactions, } from "../settings/settings"; @@ -118,7 +123,7 @@ import { constant, type Behavior } from "./Behavior"; import { enterRTCSession, getLivekitAlias, - makeFocus, + makeTransport, } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; @@ -127,6 +132,7 @@ import { type MuteStates } from "./MuteStates"; import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../widget"; +import { type Async, async, mapAsync, ready } from "./Async"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -449,27 +455,33 @@ export class CallViewModel extends ViewModel { } : undefined; - private readonly localFocus = makeFocus(this.matrixRTCSession); - - private readonly localConnection = this.localFocus.then( - (focus) => - new PublishConnection( - focus, - this.livekitAlias, - this.matrixRTCSession.room.client, - this.scope, - this.membershipsAndFocusMap$, - this.mediaDevices, - this.muteStates, - this.e2eeLivekitOptions(), - this.scope.behavior(this.trackProcessorState$), - ), - ); + private readonly join$ = new Subject(); - public readonly livekitConnectionState$ = this.scope.behavior( - combineLatest([this.localConnection]).pipe( - switchMap(([c]) => c.connectionState$), - startWith(ConnectionState.Disconnected), + public join(): void { + this.join$.next(); + } + + // This is functionally the same Observable as leave$, except here it's + // hoisted to the top of the class. This enables the cyclic dependency between + // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> + // localConnection$ -> transports$ -> joined$ -> leave$. + private readonly leaveHoisted$ = new Subject< + "user" | "timeout" | "decline" | "allOthersLeft" + >(); + + /** + * Whether we are joined to the call. This reflects our local state rather + * than whether all connections are truly up and running. + */ + private readonly joined$ = this.scope.behavior( + this.join$.pipe( + map(() => true), + // Using takeUntil with the repeat operator is perfectly valid. + // eslint-disable-next-line rxjs/no-unsafe-takeuntil + takeUntil(this.leaveHoisted$), + endWith(false), + repeat(), + startWith(false), ), ); @@ -488,125 +500,224 @@ export class CallViewModel extends ViewModel { ), ); - private readonly membershipsAndFocusMap$ = this.scope.behavior( - this.memberships$.pipe( - map((memberships) => - memberships.flatMap((m) => { - const f = this.matrixRTCSession.resolveActiveFocus(m); - return f && isLivekitTransport(f) - ? [{ membership: m, focus: f }] - : []; - }), + /** + * The transport that we would personally prefer to publish on (if not for the + * transport preferences of others, perhaps). + */ + private readonly preferredTransport = makeTransport(this.matrixRTCSession); + + /** + * Lists the transports used by ourselves, plus all other MatrixRTC session + * members. + */ + private readonly transports$: Behavior<{ + local: Async; + remote: { membership: CallMembership; transport: LivekitTransport }[]; + } | null> = this.scope.behavior( + this.joined$.pipe( + switchMap((joined) => + joined + ? combineLatest( + [ + async(this.preferredTransport), + this.memberships$, + multiSfu.value$, + ], + (preferred, memberships, multiSfu) => { + const remote = memberships.flatMap((m) => { + if (m.sender === this.userId && m.deviceId === this.deviceId) + return []; + const t = this.matrixRTCSession.resolveActiveFocus(m); + return t && isLivekitTransport(t) + ? [{ membership: m, transport: t }] + : []; + }); + let local = preferred; + if (!multiSfu) { + const oldest = this.matrixRTCSession.getOldestMembership(); + if (oldest !== undefined) { + const selection = oldest.getTransport(oldest); + if (isLivekitTransport(selection)) local = ready(selection); + } + } + return { local, remote }; + }, + ) + : of(null), ), ), ); - private readonly livekitServiceUrls$ = this.membershipsAndFocusMap$.pipe( - map((v) => new Set(v.map(({ focus }) => focus.livekit_service_url))), + /** + * Lists the transports used by each MatrixRTC session member other than + * ourselves. + */ + private readonly remoteTransports$ = this.scope.behavior( + this.transports$.pipe(map((transports) => transports?.remote ?? [])), ); + /** + * The transport over which we should be actively publishing our media. + */ + private readonly localTransport$: Behavior | null> = + this.scope.behavior( + this.transports$.pipe( + map((transports) => transports?.local ?? null), + distinctUntilChanged(deepCompare), + ), + ); + + private readonly localConnectionAndTransport$ = this.scope.behavior( + this.localTransport$.pipe( + map( + (transport) => + transport && + mapAsync(transport, (transport) => ({ + connection: new PublishConnection( + transport, + this.livekitAlias, + this.matrixRTCSession.room.client, + this.scope, + this.remoteTransports$, + this.mediaDevices, + this.muteStates, + this.e2eeLivekitOptions(), + this.scope.behavior(this.trackProcessorState$), + ), + transport, + })), + ), + ), + ); + + private readonly localConnection$ = this.scope.behavior( + this.localConnectionAndTransport$.pipe( + map((value) => value && mapAsync(value, ({ connection }) => connection)), + ), + ); + + public readonly livekitConnectionState$ = this.scope.behavior( + this.localConnection$.pipe( + switchMap((c) => + c?.state === "ready" + ? c.value.connectionState$ + : of(ConnectionState.Disconnected), + ), + ), + ); + + /** + * Connections for each transport in use by one or more session members that + * is *distinct* from the local transport. + */ private readonly remoteConnections$ = this.scope.behavior( - combineLatest([this.localFocus, this.livekitServiceUrls$]).pipe( - accumulate( - new Map(), - (prev, [localFocus, focusUrls]) => { - const stopped = new Map(prev); - const next = new Map(); - for (const focusUrl of focusUrls) { - if (focusUrl !== localFocus.livekit_service_url) { - stopped.delete(focusUrl); - - let nextConnection = prev.get(focusUrl); - if (!nextConnection) { - logger.log( - "SFU remoteConnections$ construct new connection: ", - focusUrl, - ); - nextConnection = new Connection( - { - livekit_service_url: focusUrl, - livekit_alias: this.livekitAlias, - type: "livekit", - }, - this.livekitAlias, - this.matrixRTCSession.room.client, - this.scope, - this.membershipsAndFocusMap$, - this.e2eeLivekitOptions(), - ); - } else { - logger.log( - "SFU remoteConnections$ use prev connection: ", - focusUrl, - ); - } - next.set(focusUrl, nextConnection); + this.transports$.pipe( + accumulate(new Map(), (prev, transports) => { + const next = new Map(); + + // Until the local transport becomes ready we have no idea which + // transports will actually need a dedicated remote connection + if (transports?.local.state === "ready") { + const localServiceUrl = transports.local.value.livekit_service_url; + const remoteServiceUrls = new Set( + transports.remote.flatMap(({ membership, transport }) => { + const t = this.matrixRTCSession.resolveActiveFocus(membership); + return t && + isLivekitTransport(t) && + t.livekit_service_url !== localServiceUrl + ? [t.livekit_service_url] + : []; + }), + ); + + for (const remoteServiceUrl of remoteServiceUrls) { + let nextConnection = prev.get(remoteServiceUrl); + if (!nextConnection) { + logger.log( + "SFU remoteConnections$ construct new connection: ", + remoteServiceUrl, + ); + nextConnection = new Connection( + { + livekit_service_url: remoteServiceUrl, + livekit_alias: this.livekitAlias, + type: "livekit", + }, + this.livekitAlias, + this.matrixRTCSession.room.client, + this.scope, + this.remoteTransports$, + this.e2eeLivekitOptions(), + ); + } else { + logger.log( + "SFU remoteConnections$ use prev connection: ", + remoteServiceUrl, + ); } + next.set(remoteServiceUrl, nextConnection); } + } - for (const connection of stopped.values()) connection.stop(); - return next; - }, - ), + return next; + }), + map((transports) => [...transports.values()]), ), ); - private readonly join$ = new Subject(); - - public join(): void { - this.join$.next(); - } + /** + * A list of the connections that should be active at any given time. + */ + private readonly connections$ = this.scope.behavior( + combineLatest( + [this.localConnection$, this.remoteConnections$], + (local, remote) => [ + ...(local?.state === "ready" ? [local.value] : []), + ...remote.values(), + ], + ), + ); - private readonly connectionInstructions$ = this.join$.pipe( - switchMap(() => this.remoteConnections$), - startWith(new Map()), + private readonly connectionInstructions$ = this.connections$.pipe( pairwise(), map(([prev, next]) => { const start = new Set(next.values()); - for (const connection of prev.values()) start.delete(connection); + for (const connection of prev) start.delete(connection); const stop = new Set(prev.values()); - for (const connection of next.values()) stop.delete(connection); + for (const connection of next) stop.delete(connection); return { start, stop }; }), this.scope.share, ); + /** + * Emits with a connection whenever it should be started. + */ private readonly startConnection$ = this.connectionInstructions$.pipe( concatMap(({ start }) => start), ); + /** + * Emits with a connection whenever it should be stopped. + */ private readonly stopConnection$ = this.connectionInstructions$.pipe( concatMap(({ stop }) => stop), ); public readonly allLivekitRooms$ = this.scope.behavior( - combineLatest([ - this.remoteConnections$, - this.localConnection, - this.localFocus, - ]).pipe( - map(([remoteConnections, localConnection, localFocus]) => - Array.from(remoteConnections.entries()) - .map( - ([index, c]) => - ({ - room: c.livekitRoom, - url: index, - }) as { room: LivekitRoom; url: string; isLocal?: boolean }, - ) - .concat([ - { - room: localConnection.livekitRoom, - url: localFocus.livekit_service_url, - isLocal: true, - }, - ]), + this.connections$.pipe( + map((connections) => + [...connections.values()].map((c) => ({ + room: c.livekitRoom, + url: c.transport.livekit_service_url, + isLocal: c instanceof PublishConnection, + })), ), - startWith([]), ), ); private readonly userId = this.matrixRoom.client.getUserId(); + private readonly deviceId = this.matrixRoom.client.getDeviceId(); private readonly matrixConnected$ = this.scope.behavior( // To consider ourselves connected to MatrixRTC, we check the following: @@ -679,6 +790,10 @@ export class CallViewModel extends ViewModel { // in a split-brained state. private readonly pretendToBeDisconnected$ = this.reconnecting$; + /** + * Lists, for each LiveKit room, the LiveKit participants whose media should + * be presented. + */ public readonly participantsByRoom$ = this.scope.behavior< { livekitRoom: LivekitRoom; @@ -689,9 +804,12 @@ export class CallViewModel extends ViewModel { }[]; }[] >( - combineLatest([this.localConnection, this.localFocus]) + // TODO: Move this logic into Connection/PublishConnection if possible + this.localConnectionAndTransport$ .pipe( - switchMap(([localConnection, localFocus]) => { + switchMap((values) => { + if (values?.state !== "ready") return []; + const localConnection = values.value.connection; const memberError = (): never => { throw new Error("No room member for call membership"); }; @@ -702,12 +820,9 @@ export class CallViewModel extends ViewModel { }; return this.remoteConnections$.pipe( - switchMap((connections) => + switchMap((remoteConnections) => combineLatest( - [ - [localFocus.livekit_service_url, localConnection] as const, - ...connections, - ].map(([url, c]) => + [localConnection, ...remoteConnections].map((c) => c.publishingParticipants$.pipe( map((ps) => { const participants: { @@ -726,7 +841,7 @@ export class CallViewModel extends ViewModel { return { livekitRoom: c.livekitRoom, - url, + url: c.transport.livekit_service_url, participants, }; }), @@ -809,12 +924,8 @@ export class CallViewModel extends ViewModel { * List of MediaItems that we want to display */ private readonly mediaItems$ = this.scope.behavior( - combineLatest([ - this.participantsByRoom$, - duplicateTiles.value$, - this.memberships$, - ]).pipe( - scan((prevItems, [participantsByRoom, duplicateTiles, memberships]) => { + combineLatest([this.participantsByRoom$, duplicateTiles.value$]).pipe( + scan((prevItems, [participantsByRoom, duplicateTiles]) => { const newItems: Map = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { for (const { livekitRoom, participants } of participantsByRoom) { @@ -829,6 +940,7 @@ export class CallViewModel extends ViewModel { if (prevMedia && prevMedia instanceof UserMedia) { prevMedia.updateParticipant(participant); if (prevMedia.vm.member === undefined) { + // TODO-MULTI-SFU: This is outdated. // We have a previous media created because of the `debugShowNonMember` flag. // In this case we actually replace the media item. // This "hack" never occurs if we do not use the `debugShowNonMember` debugging @@ -931,6 +1043,16 @@ export class CallViewModel extends ViewModel { this.memberships$.pipe(map((ms) => ms.length)), ); + private readonly allOthersLeft$ = this.memberships$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.every((m) => m.sender === this.userId) && + prev.some((m) => m.sender !== this.userId), + ), + map(() => {}), + ); + private readonly didSendCallNotification$ = fromEvent( this.matrixRTCSession, MatrixRTCSessionEvent.DidSendCallNotification, @@ -1055,56 +1177,12 @@ export class CallViewModel extends ViewModel { map(() => {}), throttleTime(THROTTLE_SOUND_EFFECT_MS), ); - /** - * This observable tracks the matrix users that are currently in the call. - * There can be just one matrix user with multiple participants (see also participantChanges$) - */ - public readonly matrixUserChanges$ = this.userMedia$.pipe( - map( - (mediaItems) => - new Set( - mediaItems - .map((m) => m.vm.member?.userId) - .filter((id) => id !== undefined), - ), - ), - scan< - Set, - { - userIds: Set; - joinedUserIds: Set; - leftUserIds: Set; - } - >( - (prevState, userIds) => { - const left = new Set( - [...prevState.userIds].filter((id) => !userIds.has(id)), - ); - const joined = new Set( - [...userIds].filter((id) => !prevState.userIds.has(id)), - ); - return { userIds: userIds, joinedUserIds: joined, leftUserIds: left }; - }, - { userIds: new Set(), joinedUserIds: new Set(), leftUserIds: new Set() }, - ), - ); - - private readonly allOthersLeft$ = this.matrixUserChanges$.pipe( - filter(({ userIds, leftUserIds }) => { - if (!this.userId) { - logger.warn("Could not access user ID to compute allOthersLeft"); - return false; - } - return ( - userIds.size === 1 && userIds.has(this.userId) && leftUserIds.size > 0 - ); - }), - map(() => "allOthersLeft" as const), - ); // Public for testing public readonly autoLeave$ = merge( - this.options.autoLeaveWhenOthersLeft ? this.allOthersLeft$ : NEVER, + this.options.autoLeaveWhenOthersLeft + ? this.allOthersLeft$.pipe(map(() => "allOthersLeft" as const)) + : NEVER, this.callPickupState$.pipe( filter((state) => state === "timeout" || state === "decline"), ), @@ -1132,6 +1210,9 @@ export class CallViewModel extends ViewModel { merge(this.userHangup$, this.widgetHangup$).pipe( map(() => "user" as const), ), + ).pipe( + this.scope.share, + tap((reason) => this.leaveHoisted$.next(reason)), ); /** @@ -1820,9 +1901,12 @@ export class CallViewModel extends ViewModel { * Whether we are sharing our screen. */ public readonly sharingScreen$ = this.scope.behavior( - from(this.localConnection).pipe( - switchMap((c) => sharingScreen$(c.livekitRoom.localParticipant)), - startWith(false), + from(this.localConnection$).pipe( + switchMap((c) => + c?.state === "ready" + ? sharingScreen$(c.value.livekitRoom.localParticipant) + : of(false), + ), ), ); @@ -1834,17 +1918,26 @@ export class CallViewModel extends ViewModel { "getDisplayMedia" in (navigator.mediaDevices ?? {}) && !this.urlParams.hideScreensharing ? (): void => - void this.localConnection.then( - (c) => - void c.livekitRoom.localParticipant - .setScreenShareEnabled(!this.sharingScreen$.value, { - audio: true, - selfBrowserSurface: "include", - surfaceSwitching: "include", - systemAudio: "include", - }) - .catch(logger.error), - ) + // Once a connection is ready... + void this.localConnection$ + .pipe( + takeWhile((c) => c !== null && c.state !== "error"), + switchMap((c) => (c.state === "ready" ? of(c.value) : NEVER)), + take(1), + this.scope.bind(), + ) + // ...toggle screen sharing. + .subscribe( + (c) => + void c.livekitRoom.localParticipant + .setScreenShareEnabled(!this.sharingScreen$.value, { + audio: true, + selfBrowserSurface: "include", + surfaceSwitching: "include", + systemAudio: "include", + }) + .catch(logger.error), + ) : null; public constructor( @@ -1864,32 +1957,33 @@ export class CallViewModel extends ViewModel { ) { super(); - void from(this.localConnection) - .pipe(this.scope.bind()) - .subscribe( - (c) => - void c - .start() - // eslint-disable-next-line no-console - .then(() => console.log("successfully started publishing")) - // eslint-disable-next-line no-console - .catch((e) => console.error("failed to start publishing", e)), - ); - - this.startConnection$ - .pipe(this.scope.bind()) - .subscribe((c) => void c.start()); - this.stopConnection$.pipe(this.scope.bind()).subscribe((c) => c.stop()); + // Start and stop local and remote connections as needed + this.startConnection$.pipe(this.scope.bind()).subscribe( + (c) => + void c.start().then( + () => logger.info(`Connected to ${c.transport.livekit_service_url}`), + (e) => + logger.error( + `Failed to start connection to ${c.transport.livekit_service_url}`, + e, + ), + ), + ); + this.stopConnection$.pipe(this.scope.bind()).subscribe((c) => { + logger.info(`Disconnecting from ${c.transport.livekit_service_url}`); + c.stop(); + }); - combineLatest([this.localFocus, this.join$]) - .pipe(this.scope.bind()) - .subscribe(([localFocus]) => { + // Start and stop session membership as needed + this.localTransport$.pipe(this.scope.bind()).subscribe((localTransport) => { + if (localTransport?.state === "ready") { void enterRTCSession( this.matrixRTCSession, - localFocus, + localTransport.value, this.options.encryptionSystem.kind !== E2eeType.NONE, true, true, + multiSfu.value$.value, ) .catch((e) => logger.error("Error entering RTC session", e)) .then(() => @@ -1906,19 +2000,20 @@ export class CallViewModel extends ViewModel { ), ), ); - }); - this.leave$.pipe(this.scope.bind()).subscribe(() => { - // Only sends Matrix leave event. The LiveKit session will disconnect once, uh... - // (TODO-MULTI-SFU does anything actually cause it to disconnect?) - void this.matrixRTCSession - .leaveRoomSession() - .catch((e) => logger.error("Error leaving RTC session", e)) - .then(async () => - widget?.api.transport - .send(ElementWidgetActions.HangupCall, {}) - .catch((e) => logger.error("Failed to send hangup action", e)), - ); + return (): void => + // Only sends Matrix leave event. The LiveKit session will disconnect + // as soon as either the stopConnection$ handler above gets to it or + // the view model is destroyed. + void this.matrixRTCSession + .leaveRoomSession() + .catch((e) => logger.error("Error leaving RTC session", e)) + .then(async () => + widget?.api.transport + .send(ElementWidgetActions.HangupCall, {}) + .catch((e) => logger.error("Failed to send hangup action", e)), + ); + } }); // Pause upstream of all local media tracks when we're disconnected from @@ -1927,10 +2022,12 @@ export class CallViewModel extends ViewModel { // We use matrixConnected$ rather than reconnecting$ because we want to // pause tracks during the initial joining sequence too until we're sure // that our own media is displayed on screen. - void this.localConnection.then((localConnection) => - this.matrixConnected$.pipe(this.scope.bind()).subscribe((connected) => { + combineLatest([this.localConnection$, this.matrixConnected$]) + .pipe(this.scope.bind()) + .subscribe(([connection, connected]) => { + if (connection?.state !== "ready") return; const publications = - localConnection.livekitRoom.localParticipant.trackPublications.values(); + connection.value.livekitRoom.localParticipant.trackPublications.values(); if (connected) { for (const p of publications) { if (p.track?.isUpstreamPaused === true) { @@ -1966,8 +2063,7 @@ export class CallViewModel extends ViewModel { } } } - }), - ); + }); // Join automatically this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 8eaed4635..992d88409 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -62,7 +62,7 @@ export class Connection { protected readonly sfuConfig = getSFUConfigWithOpenID( this.client, - this.focus.livekit_service_url, + this.transport.livekit_service_url, this.livekitAlias, ); @@ -72,12 +72,12 @@ export class Connection { public connectionState$: Behavior; public constructor( - protected readonly focus: LivekitTransport, + public readonly transport: LivekitTransport, protected readonly livekitAlias: string, protected readonly client: MatrixClient, protected readonly scope: ObservableScope, - protected readonly membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitTransport }[] + protected readonly remoteTransports$: Behavior< + { membership: CallMembership; transport: LivekitTransport }[] >, e2eeLivekitOptions: E2EEOptions | undefined, livekitRoom: LivekitRoom | undefined = undefined, @@ -95,12 +95,13 @@ export class Connection { this.publishingParticipants$ = this.scope.behavior( combineLatest( - [this.participantsIncludingSubscribers$, this.membershipsFocusMap$], - (participants, membershipsFocusMap) => - membershipsFocusMap + [this.participantsIncludingSubscribers$, this.remoteTransports$], + (participants, remoteTransports) => + remoteTransports // Find all members that claim to publish on this connection - .flatMap(({ membership, focus }) => - focus.livekit_service_url === this.focus.livekit_service_url + .flatMap(({ membership, transport }) => + transport.livekit_service_url === + this.transport.livekit_service_url ? [membership] : [], ) @@ -130,23 +131,35 @@ export class PublishConnection extends Connection { if (!this.stopped) await this.livekitRoom.connect(url, jwt); if (!this.stopped) { - const tracks = await this.livekitRoom.localParticipant.createTracks({ - audio: this.muteStates.audio.enabled$.value, - video: this.muteStates.video.enabled$.value, - }); - for (const track of tracks) { - await this.livekitRoom.localParticipant.publishTrack(track); + // TODO-MULTI-SFU: Prepublish a microphone track + const audio = this.muteStates.audio.enabled$.value; + const video = this.muteStates.video.enabled$.value; + // createTracks throws if called with audio=false and video=false + if (audio || video) { + const tracks = await this.livekitRoom.localParticipant.createTracks({ + audio, + video, + }); + for (const track of tracks) { + await this.livekitRoom.localParticipant.publishTrack(track); + } } } } + public stop(): void { + this.muteStates.audio.unsetHandler(); + this.muteStates.video.unsetHandler(); + super.stop(); + } + public constructor( - focus: LivekitTransport, + transport: LivekitTransport, livekitAlias: string, client: MatrixClient, scope: ObservableScope, - membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitTransport }[] + remoteTransports$: Behavior< + { membership: CallMembership; transport: LivekitTransport }[] >, devices: MediaDevices, private readonly muteStates: MuteStates, @@ -182,11 +195,11 @@ export class PublishConnection extends Connection { }); super( - focus, + transport, livekitAlias, client, scope, - membershipsFocusMap$, + remoteTransports$, e2eeLivekitOptions, room, ); @@ -218,10 +231,6 @@ export class PublishConnection extends Connection { } return this.livekitRoom.localParticipant.isCameraEnabled; }); - this.scope.onEnd(() => { - this.muteStates.audio.unsetHandler(); - this.muteStates.video.unsetHandler(); - }); const syncDevice = ( kind: MediaDeviceKind, diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index c93e88d83..07bc5665e 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -137,7 +137,7 @@ export class MuteStates { this.scope, this.mediaDevices.audioInput, this.joined$, - Config.get().media_devices.enable_video, + Config.get().media_devices.enable_audio, ); public readonly video = new MuteState( this.scope, diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index fe99d89bf..8ac816ca3 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -8,9 +8,10 @@ Please see LICENSE in the repository root for full details. import { BehaviorSubject, distinctUntilChanged, + filter, type Observable, share, - Subject, + take, takeUntil, } from "rxjs"; @@ -24,9 +25,11 @@ const nothing = Symbol("nothing"); * A scope which limits the execution lifetime of its bound Observables. */ export class ObservableScope { - private readonly ended$ = new Subject(); + private readonly ended$ = new BehaviorSubject(false); - private readonly bindImpl: MonoTypeOperator = takeUntil(this.ended$); + private readonly bindImpl: MonoTypeOperator = takeUntil( + this.ended$.pipe(filter((ended) => ended)), + ); /** * Binds an Observable to this scope, so that it completes when the scope @@ -78,15 +81,19 @@ export class ObservableScope { * Ends the scope, causing any bound Observables to complete. */ public end(): void { - this.ended$.next(); - this.ended$.complete(); + this.ended$.next(true); } /** * Register a callback to be executed when the scope is ended. */ public onEnd(callback: () => void): void { - this.ended$.subscribe(callback); + this.ended$ + .pipe( + filter((ended) => ended), + take(1), + ) + .subscribe(callback); } } diff --git a/src/utils/errors.ts b/src/utils/errors.ts index 5cb0b450a..b77c0ff0b 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -11,7 +11,7 @@ export enum ErrorCode { /** * Configuration problem due to no MatrixRTC backend/SFU is exposed via .well-known and no fallback configured. */ - MISSING_MATRIX_RTC_FOCUS = "MISSING_MATRIX_RTC_FOCUS", + MISSING_MATRIX_RTC_TRANSPORT = "MISSING_MATRIX_RTC_TRANSPORT", CONNECTION_LOST_ERROR = "CONNECTION_LOST_ERROR", /** LiveKit indicates that the server has hit its track limits */ INSUFFICIENT_CAPACITY_ERROR = "INSUFFICIENT_CAPACITY_ERROR", @@ -54,18 +54,18 @@ export class ElementCallError extends Error { } } -export class MatrixRTCFocusMissingError extends ElementCallError { +export class MatrixRTCTransportMissingError extends ElementCallError { public domain: string; public constructor(domain: string) { super( t("error.call_is_not_supported"), - ErrorCode.MISSING_MATRIX_RTC_FOCUS, + ErrorCode.MISSING_MATRIX_RTC_TRANSPORT, ErrorCategory.CONFIGURATION_ISSUE, - t("error.matrix_rtc_focus_missing", { + t("error.matrix_rtc_transport_missing", { domain, brand: import.meta.env.VITE_PRODUCT_NAME || "Element Call", - errorCode: ErrorCode.MISSING_MATRIX_RTC_FOCUS, + errorCode: ErrorCode.MISSING_MATRIX_RTC_TRANSPORT, }), ); this.domain = domain; From 1820cac3f66e9b9801209990d5ad5469ad21e97a Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 3 Oct 2025 19:14:48 -0400 Subject: [PATCH 44/45] Create media items for session members not joined to LiveKit --- src/state/CallViewModel.ts | 49 ++++++++++++++----------------------- src/state/Connection.ts | 14 +++++------ src/state/MediaViewModel.ts | 10 ++++---- src/tile/MediaView.tsx | 2 +- src/tile/SpotlightTile.tsx | 2 +- 5 files changed, 32 insertions(+), 45 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 8988e5180..6e333becb 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -304,7 +304,7 @@ class UserMedia { public readonly presenter$: Behavior; public constructor( public readonly id: string, - member: RoomMember | undefined, + member: RoomMember, participant: LocalParticipant | RemoteParticipant | undefined, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, @@ -377,7 +377,7 @@ class ScreenShare { public constructor( id: string, - member: RoomMember | undefined, + member: RoomMember, participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, @@ -799,7 +799,8 @@ export class CallViewModel extends ViewModel { livekitRoom: LivekitRoom; url: string; participants: { - participant: LocalParticipant | RemoteParticipant; + id: string; + participant: LocalParticipant | RemoteParticipant | undefined; member: RoomMember; }[]; }[] @@ -814,6 +815,7 @@ export class CallViewModel extends ViewModel { throw new Error("No room member for call membership"); }; const localParticipant = { + id: "local", participant: localConnection.livekitRoom.localParticipant, member: this.matrixRoom.getMember(this.userId ?? "") ?? memberError(), @@ -826,9 +828,14 @@ export class CallViewModel extends ViewModel { c.publishingParticipants$.pipe( map((ps) => { const participants: { - participant: LocalParticipant | RemoteParticipant; + id: string; + participant: + | LocalParticipant + | RemoteParticipant + | undefined; member: RoomMember; }[] = ps.map(({ participant, membership }) => ({ + id: `${membership.sender}:${membership.deviceId}`, participant, member: getRoomMemberFromRtcMember( @@ -929,26 +936,12 @@ export class CallViewModel extends ViewModel { const newItems: Map = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { for (const { livekitRoom, participants } of participantsByRoom) { - for (const { participant, member } of participants) { - const matrixId = participant.isLocal - ? "local" - : participant.identity; - + for (const { id, participant, member } of participants) { for (let i = 0; i < 1 + duplicateTiles; i++) { - const mediaId = `${matrixId}:${i}`; - let prevMedia = prevItems.get(mediaId); - if (prevMedia && prevMedia instanceof UserMedia) { + const mediaId = `${id}:${i}`; + const prevMedia = prevItems.get(mediaId); + if (prevMedia instanceof UserMedia) prevMedia.updateParticipant(participant); - if (prevMedia.vm.member === undefined) { - // TODO-MULTI-SFU: This is outdated. - // We have a previous media created because of the `debugShowNonMember` flag. - // In this case we actually replace the media item. - // This "hack" never occurs if we do not use the `debugShowNonMember` debugging - // option and if we always find a room member for each rtc member (which also - // only fails if we have a fundamental problem) - prevMedia = undefined; - } - } yield [ mediaId, @@ -965,14 +958,10 @@ export class CallViewModel extends ViewModel { this.mediaDevices, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( - map((m) => m.get(matrixId) ?? "[👻]"), - ), - this.handsRaised$.pipe( - map((v) => v[matrixId]?.time ?? null), - ), - this.reactions$.pipe( - map((v) => v[matrixId] ?? undefined), + map((m) => m.get(id) ?? "[👻]"), ), + this.handsRaised$.pipe(map((v) => v[id]?.time ?? null)), + this.reactions$.pipe(map((v) => v[id] ?? undefined)), ), ]; @@ -989,7 +978,7 @@ export class CallViewModel extends ViewModel { livekitRoom, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( - map((m) => m.get(matrixId) ?? "[👻]"), + map((m) => m.get(id) ?? "[👻]"), ), ), ]; diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 992d88409..4908e42f6 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -66,7 +66,7 @@ export class Connection { this.livekitAlias, ); - public readonly participantsIncludingSubscribers$; + private readonly participantsIncludingSubscribers$; public readonly publishingParticipants$; public readonly livekitRoom: LivekitRoom; @@ -105,13 +105,11 @@ export class Connection { ? [membership] : [], ) - // Find all associated publishing livekit participant objects - .flatMap((membership) => { - const participant = participants.find( - (p) => - p.identity === `${membership.sender}:${membership.deviceId}`, - ); - return participant ? [{ participant, membership }] : []; + // Pair with their associated LiveKit participant (if any) + .map((membership) => { + const id = `${membership.sender}:${membership.deviceId}`; + const participant = participants.find((p) => p.identity === id); + return { participant, membership }; }), ), [], diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index dc2c135af..016c6a499 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -255,7 +255,7 @@ abstract class BaseMediaViewModel extends ViewModel { */ // TODO: Fully separate the data layer from the UI layer by keeping the // member object internal - public readonly member: RoomMember | undefined, + public readonly member: RoomMember, // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through // livekit. protected readonly participant$: Observable< @@ -403,7 +403,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { public constructor( id: string, - member: RoomMember | undefined, + member: RoomMember, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, @@ -535,7 +535,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { public constructor( id: string, - member: RoomMember | undefined, + member: RoomMember, participant$: Behavior, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, @@ -641,7 +641,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { public constructor( id: string, - member: RoomMember | undefined, + member: RoomMember, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, @@ -736,7 +736,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { public constructor( id: string, - member: RoomMember | undefined, + member: RoomMember, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index a4fd04021..8506a6504 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -32,7 +32,7 @@ interface Props extends ComponentProps { video: TrackReferenceOrPlaceholder | undefined; videoFit: "cover" | "contain"; mirror: boolean; - member: RoomMember | undefined; + member: RoomMember; videoEnabled: boolean; unencryptedWarning: boolean; encryptionStatus: EncryptionStatus; diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 663fb9121..b1a15332f 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -55,7 +55,7 @@ interface SpotlightItemBaseProps { targetHeight: number; video: TrackReferenceOrPlaceholder | undefined; videoEnabled: boolean; - member: RoomMember | undefined; + member: RoomMember; unencryptedWarning: boolean; encryptionStatus: EncryptionStatus; displayName: string; From 1fff71ace1f11dcc869fdfc301114f9f77f67d9a Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 3 Oct 2025 21:00:45 -0400 Subject: [PATCH 45/45] Actually leave the MatrixRTC session again --- src/state/CallViewModel.ts | 119 +++++++++++++++++------------------ src/state/Connection.ts | 35 ++++++----- src/state/ObservableScope.ts | 38 +++++++++++ 3 files changed, 114 insertions(+), 78 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 6e333becb..1a20589c1 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -37,7 +37,6 @@ import { Subject, combineLatest, concat, - concatMap, distinctUntilChanged, endWith, filter, @@ -678,6 +677,9 @@ export class CallViewModel extends ViewModel { ), ); + /** + * Emits with connections whenever they should be started or stopped. + */ private readonly connectionInstructions$ = this.connections$.pipe( pairwise(), map(([prev, next]) => { @@ -688,20 +690,6 @@ export class CallViewModel extends ViewModel { return { start, stop }; }), - this.scope.share, - ); - - /** - * Emits with a connection whenever it should be started. - */ - private readonly startConnection$ = this.connectionInstructions$.pipe( - concatMap(({ start }) => start), - ); - /** - * Emits with a connection whenever it should be stopped. - */ - private readonly stopConnection$ = this.connectionInstructions$.pipe( - concatMap(({ stop }) => stop), ); public readonly allLivekitRooms$ = this.scope.behavior( @@ -1947,61 +1935,70 @@ export class CallViewModel extends ViewModel { super(); // Start and stop local and remote connections as needed - this.startConnection$.pipe(this.scope.bind()).subscribe( - (c) => - void c.start().then( - () => logger.info(`Connected to ${c.transport.livekit_service_url}`), - (e) => - logger.error( - `Failed to start connection to ${c.transport.livekit_service_url}`, - e, - ), - ), - ); - this.stopConnection$.pipe(this.scope.bind()).subscribe((c) => { - logger.info(`Disconnecting from ${c.transport.livekit_service_url}`); - c.stop(); - }); + this.connectionInstructions$ + .pipe(this.scope.bind()) + .subscribe(({ start, stop }) => { + for (const c of stop) { + logger.info(`Disconnecting from ${c.transport.livekit_service_url}`); + c.stop(); + } + for (const c of start) { + c.start().then( + () => + logger.info(`Connected to ${c.transport.livekit_service_url}`), + (e) => + logger.error( + `Failed to start connection to ${c.transport.livekit_service_url}`, + e, + ), + ); + } + }); // Start and stop session membership as needed - this.localTransport$.pipe(this.scope.bind()).subscribe((localTransport) => { + this.scope.reconcile(this.localTransport$, async (localTransport) => { if (localTransport?.state === "ready") { - void enterRTCSession( - this.matrixRTCSession, - localTransport.value, - this.options.encryptionSystem.kind !== E2eeType.NONE, - true, - true, - multiSfu.value$.value, - ) - .catch((e) => logger.error("Error entering RTC session", e)) - .then(() => - // Update our member event when our mute state changes. - this.muteStates.video.enabled$ - .pipe(this.scope.bind(), takeUntil(this.leave$)) - // eslint-disable-next-line rxjs/no-nested-subscribe - .subscribe( - (videoEnabled) => - // TODO: Ensure that these calls are serialized in case of - // fast video toggling - void this.matrixRTCSession.updateCallIntent( - videoEnabled ? "video" : "audio", - ), - ), + try { + await enterRTCSession( + this.matrixRTCSession, + localTransport.value, + this.options.encryptionSystem.kind !== E2eeType.NONE, + true, + true, + multiSfu.value$.value, ); + } catch (e) { + logger.error("Error entering RTC session", e); + } + // Update our member event when our mute state changes. + const muteSubscription = this.muteStates.video.enabled$.subscribe( + (videoEnabled) => + // TODO: Ensure that these calls are serialized in case of + // fast video toggling + void this.matrixRTCSession.updateCallIntent( + videoEnabled ? "video" : "audio", + ), + ); - return (): void => + return async (): Promise => { + muteSubscription.unsubscribe(); // Only sends Matrix leave event. The LiveKit session will disconnect // as soon as either the stopConnection$ handler above gets to it or // the view model is destroyed. - void this.matrixRTCSession - .leaveRoomSession() - .catch((e) => logger.error("Error leaving RTC session", e)) - .then(async () => - widget?.api.transport - .send(ElementWidgetActions.HangupCall, {}) - .catch((e) => logger.error("Failed to send hangup action", e)), + try { + await this.matrixRTCSession.leaveRoomSession(); + } catch (e) { + logger.error("Error leaving RTC session", e); + } + try { + await widget?.api.transport.send( + ElementWidgetActions.HangupCall, + {}, ); + } catch (e) { + logger.error("Failed to send hangup action", e); + } + }; } }); diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 4908e42f6..55afdacfc 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -125,6 +125,24 @@ export class Connection { export class PublishConnection extends Connection { public async start(): Promise { this.stopped = false; + + this.muteStates.audio.setHandler(async (desired) => { + try { + await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); + } catch (e) { + logger.error("Failed to update LiveKit audio input mute state", e); + } + return this.livekitRoom.localParticipant.isMicrophoneEnabled; + }); + this.muteStates.video.setHandler(async (desired) => { + try { + await this.livekitRoom.localParticipant.setCameraEnabled(desired); + } catch (e) { + logger.error("Failed to update LiveKit video input mute state", e); + } + return this.livekitRoom.localParticipant.isCameraEnabled; + }); + const { url, jwt } = await this.sfuConfig; if (!this.stopped) await this.livekitRoom.connect(url, jwt); @@ -213,23 +231,6 @@ export class PublishConnection extends Connection { ); trackProcessorSync(track$, trackerProcessorState$); - this.muteStates.audio.setHandler(async (desired) => { - try { - await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); - } catch (e) { - logger.error("Failed to update LiveKit audio input mute state", e); - } - return this.livekitRoom.localParticipant.isMicrophoneEnabled; - }); - this.muteStates.video.setHandler(async (desired) => { - try { - await this.livekitRoom.localParticipant.setCameraEnabled(desired); - } catch (e) { - logger.error("Failed to update LiveKit video input mute state", e); - } - return this.livekitRoom.localParticipant.isCameraEnabled; - }); - const syncDevice = ( kind: MediaDeviceKind, selected$: Observable, diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index 8ac816ca3..08a4b859b 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -7,7 +7,10 @@ Please see LICENSE in the repository root for full details. import { BehaviorSubject, + catchError, distinctUntilChanged, + EMPTY, + endWith, filter, type Observable, share, @@ -95,6 +98,41 @@ export class ObservableScope { ) .subscribe(callback); } + + // TODO-MULTI-SFU Dear Future Robin, please document this. Love, Past Robin. + public reconcile( + value$: Behavior, + callback: (value: T) => Promise<(() => Promise) | undefined>, + ): void { + let latestValue: T | typeof nothing = nothing; + let reconciledValue: T | typeof nothing = nothing; + let cleanUp: (() => Promise) | undefined = undefined; + let callbackPromise: Promise<(() => Promise) | undefined>; + value$ + .pipe( + catchError(() => EMPTY), + this.bind(), + endWith(nothing), + ) + .subscribe((value) => { + void (async (): Promise => { + if (latestValue === nothing) { + latestValue = value; + while (latestValue !== reconciledValue) { + await cleanUp?.(); + reconciledValue = latestValue; + if (latestValue !== nothing) { + callbackPromise = callback(latestValue); + cleanUp = await callbackPromise; + } + } + latestValue = nothing; + } else { + latestValue = value; + } + })(); + }); + } } /**