[WIP] Voip team/rebased multi sfu

locales/en/app.json

@@ -72,9 +72,9 @@
     "livekit_server_info": "LiveKit Server Info",
     "livekit_sfu": "LiveKit SFU: {{url}}",
     "matrix_id": "Matrix ID: {{id}}",
+    "multi_sfu": "Multi-SFU media transport",
     "mute_all_audio": "Mute all audio (participants, reactions, join sounds)",
     "show_connection_stats": "Show connection statistics",
-    "show_non_member_tiles": "Show tiles for non-member media",
     "url_params": "URL parameters",
     "use_new_membership_manager": "Use the new implementation of the call MembershipManager",
     "use_to_device_key_transport": "Use to device key transport. This will fallback to room key transport when another call member sent a room key"
@@ -92,7 +92,7 @@
     "generic_description": "Submitting debug logs will help us track down the problem.",
     "insufficient_capacity": "Insufficient capacity",
     "insufficient_capacity_description": "The server has reached its maximum capacity and you cannot join the call at this time. Try again later, or contact your server admin if the problem persists.",
-    "matrix_rtc_focus_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).",
+    "matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).",
     "open_elsewhere": "Opened in another tab",
     "open_elsewhere_description": "{{brand}} has been opened in another tab. If that doesn't sound right, try reloading the page.",
     "room_creation_restricted": "Failed to create call",

package.json

@@ -108,7 +108,7 @@
     "livekit-client": "^2.13.0",
     "lodash-es": "^4.17.21",
     "loglevel": "^1.9.1",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=develop",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU",
     "matrix-widget-api": "^1.13.0",
     "normalize.css": "^8.0.1",
     "observable-hooks": "^4.2.3",

src/livekit/MatrixAudioRenderer.test.tsx

@@ -18,7 +18,7 @@ import { useTracks } from "@livekit/components-react";
 
 import { testAudioContext } from "../useAudioContext.test";
 import * as MediaDevicesContext from "../MediaDevicesContext";
-import { MatrixAudioRenderer } from "./MatrixAudioRenderer";
+import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer";
 import { mockMediaDevices, mockTrack } from "../utils/test";
 
 export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
@@ -54,7 +54,7 @@ vi.mocked(useTracks).mockReturnValue(tracks);
 it("should render for member", () => {
   const { container, queryAllByTestId } = render(
     <MediaDevicesProvider value={mockMediaDevices({})}>
-      <MatrixAudioRenderer
+      <LivekitRoomAudioRenderer
         members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
       />
     </MediaDevicesProvider>,
@@ -69,7 +69,7 @@ it("should not render without member", () => {
   ] as CallMembership[];
   const { container, queryAllByTestId } = render(
     <MediaDevicesProvider value={mockMediaDevices({})}>
-      <MatrixAudioRenderer members={memberships} />
+      <LivekitRoomAudioRenderer members={memberships} />
     </MediaDevicesProvider>,
   );
   expect(container).toBeTruthy();
@@ -79,7 +79,7 @@ it("should not render without member", () => {
 it("should not setup audioContext gain and pan if there is no need to.", () => {
   render(
     <MediaDevicesProvider value={mockMediaDevices({})}>
-      <MatrixAudioRenderer
+      <LivekitRoomAudioRenderer
         members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
       />
     </MediaDevicesProvider>,
@@ -102,7 +102,7 @@ it("should setup audioContext gain and pan", () => {
   });
   render(
     <MediaDevicesProvider value={mockMediaDevices({})}>
-      <MatrixAudioRenderer
+      <LivekitRoomAudioRenderer
         members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
       />
     </MediaDevicesProvider>,

src/livekit/MatrixAudioRenderer.tsx

@@ -6,27 +6,36 @@
 */
 
 import { getTrackReferenceId } from "@livekit/components-core";
+import { type Room as LivekitRoom, type Participant } from "livekit-client";
 import { type RemoteAudioTrack, Track } from "livekit-client";
 import { useEffect, useMemo, useRef, useState, type ReactNode } from "react";
 import {
   useTracks,
   AudioTrack,
   type AudioTrackProps,
 } from "@livekit/components-react";
-import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
+import { type RoomMember } from "matrix-js-sdk";
 import { logger } from "matrix-js-sdk/lib/logger";
 
 import { useEarpieceAudioConfig } from "../MediaDevicesContext";
 import { useReactiveState } from "../useReactiveState";
 import * as controls from "../controls";
-
+import {} from "@livekit/components-core";
 export interface MatrixAudioRendererProps {
+  /**
+   * The service URL of the LiveKit room.
+   */
+  url: string;
+  livekitRoom: LivekitRoom;
   /**
    * The list of participants to render audio for.
    * This list needs to be composed based on the matrixRTC members so that we do not play audio from users
    * that are not expected to be in the rtc session.
    */
-  members: CallMembership[];
+  participants: {
+    participant: Participant;
+    member: RoomMember;
+  }[];
   /**
    * If set to `true`, mutes all audio tracks rendered by the component.
    * @remarks
@@ -49,14 +58,15 @@
  * ```
  * @public
  */
-export function MatrixAudioRenderer({
-  members,
+export function LivekitRoomAudioRenderer({
+  url,
+  livekitRoom,
+  participants,
   muted,
 }: MatrixAudioRendererProps): ReactNode {
-  const validIdentities = useMemo(
-    () =>
-      new Set(members?.map((member) => `${member.sender}:${member.deviceId}`)),
-    [members],
+  const participantSet = useMemo(
+    () => new Set(participants.map(({ participant }) => participant)),
+    [participants],
   );
 
   const loggedInvalidIdentities = useRef(new Set<string>());
@@ -68,11 +78,11 @@
    * @param identity The identity of the track that is invalid
    * @param validIdentities The list of valid identities
    */
-  const logInvalid = (identity: string, validIdentities: Set<string>): void => {
+  const logInvalid = (identity: string): void => {
     if (loggedInvalidIdentities.current.has(identity)) return;
     logger.warn(
-      `[MatrixAudioRenderer] Audio track ${identity} has no matching matrix call member`,
-      `current members: ${Array.from(validIdentities.values())}`,
+      `[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`,
+      `current members: ${participants.map((p) => p.participant.identity)}`,
       `track will not get rendered`,
     );
     loggedInvalidIdentities.current.add(identity);
@@ -87,25 +97,30 @@
     {
       updateOnlyOn: [],
       onlySubscribed: true,
+      room: livekitRoom,
     },
   ).filter((ref) => {
-    const isValid = validIdentities?.has(ref.participant.identity);
+    const isValid = participantSet?.has(ref.participant);
     if (!isValid && !ref.participant.isLocal)
-      logInvalid(ref.participant.identity, validIdentities);
+      logInvalid(ref.participant.identity);
     return (
       !ref.participant.isLocal &&
       ref.publication.kind === Track.Kind.Audio &&
       isValid
     );
   });
+
   useEffect(() => {
-    if (!tracks.some((t) => !validIdentities.has(t.participant.identity))) {
+    if (
+      loggedInvalidIdentities.current.size &&
+      tracks.every((t) => participantSet.has(t.participant))
+    ) {
       logger.debug(
-        `[MatrixAudioRenderer] All audio tracks have a matching matrix call member identity.`,
+        `[MatrixAudioRenderer] All audio tracks from ${url} have a matching matrix call member identity.`,
       );
       loggedInvalidIdentities.current.clear();
     }
-  }, [tracks, validIdentities]);
+  }, [tracks, participantSet, url]);
 
   // This component is also (in addition to the "only play audio for connected members" logic above)
   // responsible for mimicking earpiece audio on iPhones.

src/livekit/TrackProcessorContext.tsx

@@ -19,14 +19,21 @@ import {
   useMemo,
 } from "react";
 import { type LocalVideoTrack } from "livekit-client";
+import { combineLatest, map, type Observable } from "rxjs";
+import { useObservable } from "observable-hooks";
 
 import {
   backgroundBlur as backgroundBlurSettings,
   useSetting,
 } from "../settings/settings";
 import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer";
+import { type Behavior } from "../state/Behavior";
 
-type ProcessorState = {
+//TODO-MULTI-SFU: This is not yet fully there.
+// it is a combination of exposing observable and react hooks.
+// preferably we should not make this a context anymore and instead just a vm?
+
+export type ProcessorState = {
   supported: boolean | undefined;
   processor: undefined | ProcessorWrapper<BackgroundOptions>;
 };
@@ -42,6 +49,39 @@ export function useTrackProcessor(): ProcessorState {
   return state;
 }
 
+export function useTrackProcessorObservable$(): Observable<ProcessorState> {
+  const state = use(ProcessorContext);
+  if (state === undefined)
+    throw new Error(
+      "useTrackProcessor must be used within a ProcessorProvider",
+    );
+  const state$ = useObservable(
+    (init$) => init$.pipe(map(([init]) => init)),
+    [state],
+  );
+
+  return state$;
+}
+
+export const trackProcessorSync = (
+  videoTrack$: Behavior<LocalVideoTrack | null>,
+  processor$: Behavior<ProcessorState>,
+): void => {
+  combineLatest([videoTrack$, processor$]).subscribe(
+    ([videoTrack, processorState]) => {
+      if (!processorState) return;
+      if (!videoTrack) return;
+      const { processor } = processorState;
+      if (processor && !videoTrack.getProcessor()) {
+        void videoTrack.setProcessor(processor);
+      }
+      if (!processor && videoTrack.getProcessor()) {
+        void videoTrack.stopProcessor();
+      }
+    },
+  );
+};
+
 export const useTrackProcessorSync = (
   videoTrack: LocalVideoTrack | null,
 ): void => {

src/livekit/openIDSFU.ts

@@ -7,12 +7,7 @@ Please see LICENSE in the repository root for full details.
 
 import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk";
 import { logger } from "matrix-js-sdk/lib/logger";
-import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
-import { useEffect, useState } from "react";
-import { type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc";
 
-import { useActiveLivekitFocus } from "../room/useActiveFocus";
-import { useErrorBoundary } from "../useErrorBoundary";
 import { FailToGetOpenIdToken } from "../utils/errors";
 import { doNetworkOperationWithRetry } from "../utils/matrix";
 
@@ -34,38 +29,11 @@ export type OpenIDClientParts = Pick<
   "getOpenIdToken" | "getDeviceId"
 >;
 
-export function useOpenIDSFU(
-  client: OpenIDClientParts,
-  rtcSession: MatrixRTCSession,
-): SFUConfig | undefined {
-  const [sfuConfig, setSFUConfig] = useState<SFUConfig | undefined>(undefined);
-
-  const activeFocus = useActiveLivekitFocus(rtcSession);
-  const { showErrorBoundary } = useErrorBoundary();
-
-  useEffect(() => {
-    if (activeFocus) {
-      getSFUConfigWithOpenID(client, activeFocus).then(
-        (sfuConfig) => {
-          setSFUConfig(sfuConfig);
-        },
-        (e) => {
-          showErrorBoundary(new FailToGetOpenIdToken(e));
-          logger.error("Failed to get SFU config", e);
-        },
-      );
-    } else {
-      setSFUConfig(undefined);
-    }
-  }, [client, activeFocus, showErrorBoundary]);
-
-  return sfuConfig;
-}
-
 export async function getSFUConfigWithOpenID(
   client: OpenIDClientParts,
-  activeFocus: LivekitFocus,
-): Promise<SFUConfig | undefined> {
+  serviceUrl: string,
+  livekitAlias: string,
+): Promise<SFUConfig> {
   let openIdToken: IOpenIDToken;
   try {
     openIdToken = await doNetworkOperationWithRetry(async () =>
@@ -78,26 +46,16 @@ export async function getSFUConfigWithOpenID(
   }
   logger.debug("Got openID token", openIdToken);
 
-  try {
-    logger.info(
-      `Trying to get JWT from call's active focus URL of ${activeFocus.livekit_service_url}...`,
-    );
-    const sfuConfig = await getLiveKitJWT(
-      client,
-      activeFocus.livekit_service_url,
-      activeFocus.livekit_alias,
-      openIdToken,
-    );
-    logger.info(`Got JWT from call's active focus URL.`);
+  logger.info(`Trying to get JWT for focus ${serviceUrl}...`);
+  const sfuConfig = await getLiveKitJWT(
+    client,
+    serviceUrl,
+    livekitAlias,
+    openIdToken,
+  );
+  logger.info(`Got JWT from call's active focus URL.`);
 
-    return sfuConfig;
-  } catch (e) {
-    logger.warn(
-      `Failed to get JWT from RTC session's active focus URL of ${activeFocus.livekit_service_url}.`,
-      e,
-    );
-    return undefined;
-  }
+  return sfuConfig;
 }
 
 async function getLiveKitJWT(

src/livekit/useECConnectionState.ts

@@ -1,3 +1,4 @@
+// TODO not used anymore - remove
 /*
 Copyright 2023, 2024 New Vector Ltd.