Connection State: Error Handling + Cleaning / refactoring /

locales/en/app.json

@@ -72,6 +72,7 @@
     "livekit_server_info": "LiveKit Server Info",
     "livekit_sfu": "LiveKit SFU: {{url}}",
     "matrix_id": "Matrix ID: {{id}}",
+    "multi_sfu": "Multi-SFU media transport",
     "mute_all_audio": "Mute all audio (participants, reactions, join sounds)",
     "show_connection_stats": "Show connection statistics",
     "url_params": "URL parameters",
@@ -91,7 +92,7 @@
     "generic_description": "Submitting debug logs will help us track down the problem.",
     "insufficient_capacity": "Insufficient capacity",
     "insufficient_capacity_description": "The server has reached its maximum capacity and you cannot join the call at this time. Try again later, or contact your server admin if the problem persists.",
-    "matrix_rtc_focus_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).",
+    "matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).",
     "open_elsewhere": "Opened in another tab",
     "open_elsewhere_description": "{{brand}} has been opened in another tab. If that doesn't sound right, try reloading the page.",
     "room_creation_restricted": "Failed to create call",

package.json

@@ -99,6 +99,7 @@
     "eslint-plugin-react-hooks": "^5.0.0",
     "eslint-plugin-rxjs": "^5.0.3",
     "eslint-plugin-unicorn": "^56.0.0",
+    "fetch-mock": "11.1.5",
     "global-jsdom": "^26.0.0",
     "i18next": "^24.0.0",
     "i18next-browser-languagedetector": "^8.0.0",

src/livekit/MatrixAudioRenderer.test.tsx

@@ -6,20 +6,29 @@ Please see LICENSE in the repository root for full details.
 */
 
 import { afterEach, beforeEach, expect, it, vi } from "vitest";
-import { render } from "@testing-library/react";
-import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
+import { render, type RenderResult } from "@testing-library/react";
 import {
   getTrackReferenceId,
   type TrackReference,
 } from "@livekit/components-core";
-import { type RemoteAudioTrack } from "livekit-client";
+import {
+  type Participant,
+  type RemoteAudioTrack,
+  type RemoteParticipant,
+  type Room,
+} from "livekit-client";
 import { type ReactNode } from "react";
 import { useTracks } from "@livekit/components-react";
 
 import { testAudioContext } from "../useAudioContext.test";
 import * as MediaDevicesContext from "../MediaDevicesContext";
 import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer";
-import { mockMediaDevices, mockTrack } from "../utils/test";
+import {
+  mockMatrixRoomMember,
+  mockMediaDevices,
+  mockRtcMembership,
+  mockTrack,
+} from "../utils/test";
 
 export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
 
@@ -48,42 +57,148 @@ vi.mock("@livekit/components-react", async (importOriginal) => {
   };
 });
 
-const tracks = [mockTrack("test:123")];
-vi.mocked(useTracks).mockReturnValue(tracks);
-
-it("should render for member", () => {
-  const { container, queryAllByTestId } = render(
+let tracks: TrackReference[] = [];
+
+/**
+ * Render the test component with given rtc members and livekit participant identities.
+ *
+ * It is possible to have rtc members that are not in livekit (e.g. not yet joined) and vice versa.
+ *
+ * @param rtcMembers - Array of active rtc members with userId and deviceId.
+ * @param livekitParticipantIdentities - Array of livekit participant (that are publishing).
+ * */
+
+function renderTestComponent(
+  rtcMembers: { userId: string; deviceId: string }[],
+  livekitParticipantIdentities: ({ id: string; isLocal?: boolean } | string)[],
+): RenderResult {
+  const liveKitParticipants = livekitParticipantIdentities.map((p) => {
+    const identity = typeof p === "string" ? p : p.id;
+    const isLocal = typeof p === "string" ? false : (p.isLocal ?? false);
+    return vi.mocked<RemoteParticipant>({
+      identity,
+      isLocal,
+    } as unknown as RemoteParticipant);
+  });
+  const participants = rtcMembers.map(({ userId, deviceId }) => {
+    const p = liveKitParticipants.find(
+      (p) => p.identity === `${userId}:${deviceId}`,
+    );
+    const localRtcMember = mockRtcMembership(userId, deviceId);
+    const member = mockMatrixRoomMember(localRtcMember);
+    return {
+      id: `${userId}:${deviceId}`,
+      participant: p,
+      member,
+    };
+  });
+  const livekitRoom = vi.mocked<Room>({
+    remoteParticipants: new Map<string, Participant>(
+      liveKitParticipants.map((p) => [p.identity, p]),
+    ),
+  } as unknown as Room);
+
+  tracks = participants
+    .filter((p) => p.participant)
+    .map((p) => mockTrack(p.participant!)) as TrackReference[];
+
+  vi.mocked(useTracks).mockReturnValue(tracks);
+  return render(
     <MediaDevicesProvider value={mockMediaDevices({})}>
       <LivekitRoomAudioRenderer
-        members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
+        participants={participants}
+        livekitRoom={livekitRoom}
+        url={""}
       />
     </MediaDevicesProvider>,
   );
+}
+
+it("should render for member", () => {
+  const { container, queryAllByTestId } = renderTestComponent(
+    [{ userId: "@alice", deviceId: "DEV0" }],
+    ["@alice:DEV0"],
+  );
   expect(container).toBeTruthy();
   expect(queryAllByTestId("audio")).toHaveLength(1);
 });
 
 it("should not render without member", () => {
-  const memberships = [
-    { sender: "othermember", deviceId: "123" },
-  ] as CallMembership[];
-  const { container, queryAllByTestId } = render(
-    <MediaDevicesProvider value={mockMediaDevices({})}>
-      <LivekitRoomAudioRenderer members={memberships} />
-    </MediaDevicesProvider>,
+  const { container, queryAllByTestId } = renderTestComponent(
+    [{ userId: "@bob", deviceId: "DEV0" }],
+    ["@alice:DEV0"],
   );
   expect(container).toBeTruthy();
   expect(queryAllByTestId("audio")).toHaveLength(0);
 });
 
+const TEST_CASES: {
+  rtcUsers: { userId: string; deviceId: string }[];
+  livekitParticipantIdentities: (string | { id: string; isLocal?: boolean })[];
+  expectedAudioTracks: number;
+}[] = [
+  {
+    rtcUsers: [
+      { userId: "@alice", deviceId: "DEV0" },
+      { userId: "@alice", deviceId: "DEV1" },
+      { userId: "@bob", deviceId: "DEV0" },
+    ],
+    livekitParticipantIdentities: [
+      { id: "@alice:DEV0" },
+      "@bob:DEV0",
+      "@alice:DEV1",
+    ],
+    expectedAudioTracks: 3,
+  },
+  // Alice DEV0 is local participant, should not render
+  {
+    rtcUsers: [
+      { userId: "@alice", deviceId: "DEV0" },
+      { userId: "@alice", deviceId: "DEV1" },
+      { userId: "@bob", deviceId: "DEV0" },
+    ],
+    livekitParticipantIdentities: [
+      { id: "@alice:DEV0", isLocal: true },
+      "@bob:DEV0",
+      "@alice:DEV1",
+    ],
+    expectedAudioTracks: 2,
+  },
+  // Charlie is a rtc member but not in livekit
+  {
+    rtcUsers: [
+      { userId: "@alice", deviceId: "DEV0" },
+      { userId: "@bob", deviceId: "DEV0" },
+      { userId: "@charlie", deviceId: "DEV0" },
+    ],
+    livekitParticipantIdentities: ["@alice:DEV0", { id: "@bob:DEV0" }],
+    expectedAudioTracks: 2,
+  },
+  // Charlie is in livekit but not rtc member
+  {
+    rtcUsers: [
+      { userId: "@alice", deviceId: "DEV0" },
+      { userId: "@bob", deviceId: "DEV0" },
+    ],
+    livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@charlie:DEV0"],
+    expectedAudioTracks: 2,
+  },
+];
+
+TEST_CASES.forEach(
+  ({ rtcUsers, livekitParticipantIdentities, expectedAudioTracks }, index) => {
+    it(`should render sound test cases #${index + 1}`, () => {
+      const { queryAllByTestId } = renderTestComponent(
+        rtcUsers,
+        livekitParticipantIdentities,
+      );
+      expect(queryAllByTestId("audio")).toHaveLength(expectedAudioTracks);
+    });
+  },
+);
+
 it("should not setup audioContext gain and pan if there is no need to.", () => {
-  render(
-    <MediaDevicesProvider value={mockMediaDevices({})}>
-      <LivekitRoomAudioRenderer
-        members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
-      />
-    </MediaDevicesProvider>,
-  );
+  renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]);
   const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
 
   expect(audioTrack.setAudioContext).toHaveBeenCalledTimes(1);
@@ -100,13 +215,8 @@ it("should setup audioContext gain and pan", () => {
     pan: 1,
     volume: 0.1,
   });
-  render(
-    <MediaDevicesProvider value={mockMediaDevices({})}>
-      <LivekitRoomAudioRenderer
-        members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
-      />
-    </MediaDevicesProvider>,
-  );
+
+  renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]);
 
   const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
   expect(audioTrack.setAudioContext).toHaveBeenCalled();

src/livekit/MatrixAudioRenderer.tsx

@@ -32,8 +32,11 @@ export interface MatrixAudioRendererProps {
    * This list needs to be composed based on the matrixRTC members so that we do not play audio from users
    * that are not expected to be in the rtc session.
    */
+  // TODO: Why do we have this structure? looks like we only need the valid/active participants (not the room member or id)?
   participants: {
-    participant: Participant;
+    id: string;
+    // TODO it appears to be optional as per InCallView? but what does that mean here? a rtc member not yet joined in livekit?
+    participant: Participant | undefined;
     member: RoomMember;
   }[];
   /**
@@ -64,8 +67,15 @@ export function LivekitRoomAudioRenderer({
   participants,
   muted,
 }: MatrixAudioRendererProps): ReactNode {
-  const participantSet = useMemo(
-    () => new Set(participants.map(({ participant }) => participant)),
+  // This is the list of valid identities that are allowed to play audio.
+  // It is derived from the list of matrix rtc members.
+  const validIdentities = useMemo(
+    () =>
+      new Set(
+        participants
+          .filter(({ participant }) => participant) // filter out participants that are not yet joined in livekit
+          .map(({ participant }) => participant!.identity),
+      ),
     [participants],
   );
 
@@ -82,7 +92,7 @@ export function LivekitRoomAudioRenderer({
     if (loggedInvalidIdentities.current.has(identity)) return;
     logger.warn(
       `[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`,
-      `current members: ${participants.map((p) => p.participant.identity)}`,
+      `current members: ${participants.map((p) => p.participant?.identity)}`,
       `track will not get rendered`,
     );
     loggedInvalidIdentities.current.add(identity);
@@ -100,7 +110,7 @@ export function LivekitRoomAudioRenderer({
       room: livekitRoom,
     },
   ).filter((ref) => {
-    const isValid = participantSet?.has(ref.participant);
+    const isValid = validIdentities.has(ref.participant.identity);
     if (!isValid && !ref.participant.isLocal)
       logInvalid(ref.participant.identity);
     return (
@@ -113,14 +123,14 @@ export function LivekitRoomAudioRenderer({
   useEffect(() => {
     if (
       loggedInvalidIdentities.current.size &&
-      tracks.every((t) => participantSet.has(t.participant))
+      tracks.every((t) => validIdentities.has(t.participant.identity))
     ) {
       logger.debug(
         `[MatrixAudioRenderer] All audio tracks from ${url} have a matching matrix call member identity.`,
       );
       loggedInvalidIdentities.current.clear();
     }
-  }, [tracks, participantSet, url]);
+  }, [tracks, validIdentities, url]);
 
   // This component is also (in addition to the "only play audio for connected members" logic above)
   // responsible for mimicking earpiece audio on iPhones.

src/main.tsx

@@ -11,7 +11,6 @@ Please see LICENSE in the repository root for full details.
 // dependency references.
 import "matrix-js-sdk/lib/browser-index";
 
-import { StrictMode } from "react";
 import { createRoot } from "react-dom/client";
 import "./index.css";
 import { logger } from "matrix-js-sdk/lib/logger";

src/room/CallEventAudioRenderer.test.tsx

@@ -155,7 +155,8 @@ test("plays one sound when a hand is raised", () => {
 
   act(() => {
     handRaisedSubject$.next({
-      [bobRtcMember.callId]: {
+      // TODO: What is this string supposed to be?
+      [`${bobRtcMember.sender}:${bobRtcMember.deviceId}`]: {
         time: new Date(),
         membershipEventId: "",
         reactionEventId: "",

src/room/GroupCallErrorBoundary.test.tsx

@@ -26,15 +26,15 @@ import {
   E2EENotSupportedError,
   type ElementCallError,
   InsufficientCapacityError,
-  MatrixRTCFocusMissingError,
+  MatrixRTCTransportMissingError,
   UnknownCallError,
 } from "../utils/errors.ts";
 import { mockConfig } from "../utils/test.ts";
 import { ElementWidgetActions, type WidgetHelpers } from "../widget.ts";
 
 test.each([
   {
-    error: new MatrixRTCFocusMissingError("example.com"),
+    error: new MatrixRTCTransportMissingError("example.com"),
     expectedTitle: "Call is not supported",
   },
   {
@@ -85,7 +85,7 @@ test.each([
 );
 
 test("should render the error page with link back to home", async () => {
-  const error = new MatrixRTCFocusMissingError("example.com");
+  const error = new MatrixRTCTransportMissingError("example.com");
   const TestComponent = (): ReactNode => {
     throw error;
   };
@@ -213,7 +213,7 @@ describe("Rageshake button", () => {
 });
 
 test("should have a close button in widget mode", async () => {
-  const error = new MatrixRTCFocusMissingError("example.com");
+  const error = new MatrixRTCTransportMissingError("example.com");
   const TestComponent = (): ReactNode => {
     throw error;
   };