Some potential sync request spam fixes

.changeset/fix_background_sync.md

@@ -0,0 +1,5 @@
+---
+default: patch
+---
+
+Some fixes to sync requests being spammed on loading screen and for multi-account background syncing, it should also load faster now!

src/app/pages/client/BackgroundNotifications.tsx

@@ -3,6 +3,7 @@ import type { MatrixClient, MatrixEvent, Room } from '$types/matrix-sdk';
 import {
   ClientEvent,
   createClient,
+  IndexedDBStore,
   MatrixEventEvent,
   RoomEvent,
   SyncState,
@@ -47,25 +48,47 @@ import { mobileOrTablet } from '$utils/user-agent';
 
 const log = createLogger('BackgroundNotifications');
 const debugLog = createDebugLogger('BackgroundNotifications');
+
+const BACKGROUND_SYNC_POLL_TIMEOUT_MS = 60_000;
+const BACKGROUND_STAGGER_DELAY_MS = 5_000;
+
 const isClientReadyForNotifications = (state: SyncState | string | null): boolean =>
   state === SyncState.Prepared || state === SyncState.Syncing || state === SyncState.Catchup;
 
 const startBackgroundClient = async (
   session: Session,
   slidingSyncConfig: ReturnType<typeof useClientConfig>['slidingSync']
 ): Promise<MatrixClient> => {
+  const storeName = {
+    sync: `bg-sync${session.userId}`,
+    crypto: `bg-crypto${session.userId}`,
+    rustCryptoPrefix: `bg-sync${session.userId}`,
+  };
+
+  const indexedDBStore = new IndexedDBStore({
+    indexedDB: global.indexedDB,
+    localStorage: global.localStorage,
+    dbName: storeName.sync,
+  });
+
   const mx = createClient({
     baseUrl: session.baseUrl,
     accessToken: session.accessToken,
     userId: session.userId,
     deviceId: session.deviceId,
+    store: indexedDBStore,
     timelineSupport: false,
   });
-  await startClient(mx, {
+
+  const startOpts = {
     baseUrl: session.baseUrl,
-    slidingSync: slidingSyncConfig,
+    slidingSync: session.slidingSyncOptIn ? slidingSyncConfig : undefined,
     sessionSlidingSyncOptIn: session.slidingSyncOptIn,
-  });
+    pollTimeoutMs: BACKGROUND_SYNC_POLL_TIMEOUT_MS,
+    timelineLimit: 1,
+  };
+
+  await startClient(mx, startOpts);
   return mx;
 };
 
@@ -136,9 +159,11 @@ export function BackgroundNotifications() {
   // listeners before stopping a background client.
   const clientCleanupRef = useRef(new Map());
 
+  const activeUserId = activeSessionId ?? sessions[0]?.userId;
+
   const inactiveSessions = useMemo(
-    () => sessions.filter((s) => s.userId !== (activeSessionId ?? sessions[0]?.userId)),
-    [sessions, activeSessionId]
+    () => sessions.filter((s) => s.userId !== activeUserId),
+    [sessions, activeUserId]
   );
   // Ref so retry setTimeout callbacks can access the current session list
   // without stale closures.
@@ -540,20 +565,30 @@ export function BackgroundNotifications() {
         });
     };
 
-    inactiveSessions.forEach((session) => {
-      if (!current.has(session.userId)) startSession(session);
+    const pendingSessions = inactiveSessions.filter((s) => !current.has(s.userId));
+    const staggerTimers: ReturnType<typeof setTimeout>[] = [];
+    pendingSessions.forEach((session, idx) => {
+      if (idx === 0) {
+        startSession(session);
+      } else {
+        staggerTimers.push(
+          setTimeout(() => startSession(session), idx * BACKGROUND_STAGGER_DELAY_MS)
+        );
+      }
     });
 
-    // Capture the cleanup map for this effect instance so teardown runs against
-    // the listeners registered while this effect was active.
     const cleanupMap = clientCleanupRef.current;
+    const activeUserIds = new Set(inactiveSessions.map((s) => s.userId));
     return () => {
+      staggerTimers.forEach(clearTimeout);
       current.forEach((mx, userId) => {
-        cleanupMap.get(userId)?.();
-        cleanupMap.delete(userId);
-        stopClient(mx);
+        if (!activeUserIds.has(userId)) {
+          cleanupMap.get(userId)?.();
+          cleanupMap.delete(userId);
+          stopClient(mx);
+          current.delete(userId);
+        }
       });
-      current.clear();
     };
   }, [
     clientConfig.slidingSync,

src/app/utils/msc4459helper.ts

@@ -47,7 +47,7 @@ function getImagePackReferencesForMxcInternal(
     .map((pack) => {
       const img = pack.getImages(imageUsage).find((val) => val.url === mxcUrl);
       const room = matrixClient.getRoom(pack.address?.roomId);
-      if (!room || (isRoomPrivate(matrixClient, room) && !bypassPrivateFilter)) return;
+      if (!room || (isRoomPrivate(matrixClient, room) && !bypassPrivateFilter)) return undefined;
       const viaServers = new SerializableSet<string>();
       if (room)
         getViaServers(room).forEach((via) => {

src/client/initMatrix.ts

@@ -2,6 +2,7 @@ import type { CryptoCallbacks, MatrixClient, ISyncStateData } from '$types/matri
 import {
   ClientEvent,
   createClient,
+  Filter,
   IndexedDBStore,
   IndexedDBCryptoStore,
   SyncState,
@@ -26,7 +27,7 @@ const classicSyncObserverByClient = new WeakMap<
   MatrixClient,
   (state: SyncState, prevState: SyncState | null, data?: ISyncStateData) => void
 >();
-const FAST_SYNC_POLL_TIMEOUT_MS = 10000;
+const FAST_SYNC_POLL_TIMEOUT_MS = 30_000;
 const SLIDING_SYNC_POLL_TIMEOUT_MS = 20000;
 type SyncTransport = 'classic' | 'sliding';
 type SyncTransportReason =
@@ -47,8 +48,54 @@ type SyncTransportMeta = {
   reason: SyncTransportReason;
 };
 const syncTransportByClient = new WeakMap<MatrixClient, SyncTransportMeta>();
+const fetchRoomEventStartupCleanupByClient = new WeakMap<MatrixClient, () => void>();
 const COLD_CACHE_BOOTSTRAP_TIMEOUT_MS = 20000;
 
+type FetchRoomEventResult = Awaited<ReturnType<MatrixClient['fetchRoomEvent']>>;
+type MatrixClientWithWritableFetchRoomEvent = MatrixClient & {
+  fetchRoomEvent: (roomId: string, eventId: string) => Promise<FetchRoomEventResult>;
+};
+
+// Replace fetchRoomEvent for first sync so thread roots don't each trigger GET /event
+// Uses cached timeline events when present otherwise a stub that fetches when user opens thread
+function installStartupFetchRoomEventPatch(mx: MatrixClient): void {
+  fetchRoomEventStartupCleanupByClient.get(mx)?.();
+
+  const mxWritable = mx as MatrixClientWithWritableFetchRoomEvent;
+  const origFetchRoomEvent = mx.fetchRoomEvent.bind(mx);
+  let restored = false;
+
+  const restore = () => {
+    if (restored) return;
+    restored = true;
+    fetchRoomEventStartupCleanupByClient.delete(mx);
+    // Put the real fetchRoomEvent back and detach this
+    mxWritable.fetchRoomEvent = origFetchRoomEvent;
+    mx.off(ClientEvent.Sync, onSync);
+  };
+
+  const onSync = (state: SyncState) => {
+    // Initial sync burst is over, let normal server fetches run again
+    if (state === SyncState.Prepared || state === SyncState.Syncing) {
+      restore();
+    }
+  };
+
+  mxWritable.fetchRoomEvent = (roomId: string, eventId: string) => {
+    if (restored) return origFetchRoomEvent(roomId, eventId);
+    const cachedEvent = mx.getRoom(roomId)?.findEventById(eventId);
+    // Reuse sync payload instead of another GET when we already have the root.
+    const payload: FetchRoomEventResult = cachedEvent?.event ?? {
+      event_id: eventId,
+      room_id: roomId,
+    };
+    return Promise.resolve(payload);
+  };
+
+  mx.on(ClientEvent.Sync, onSync);
+  fetchRoomEventStartupCleanupByClient.set(mx, restore);
+}
+
 export const resolveSlidingEnabled = (enabled: SlidingSyncConfig['enabled']): boolean => {
   if (enabled === undefined) return false;
   if (typeof enabled === 'boolean') return enabled;
@@ -315,7 +362,9 @@ export const initClient = async (session: Session): Promise<MatrixClient> => {
 
   const wipeAllStores = async () => {
     log.warn('initClient: wiping all stores for', session.userId);
-    debugLog.warn('sync', 'Wiping all stores due to mismatch', { userId: session.userId });
+    debugLog.warn('sync', 'Wiping all stores due to mismatch', {
+      userId: session.userId,
+    });
     Sentry.addBreadcrumb({
       category: 'crypto',
       message: 'Crypto store mismatch — wiping local stores and retrying',
@@ -353,18 +402,24 @@ export const initClient = async (session: Session): Promise<MatrixClient> => {
   }
 
   try {
-    await mx.initRustCrypto({ cryptoDatabasePrefix: storeName.rustCryptoPrefix });
+    await mx.initRustCrypto({
+      cryptoDatabasePrefix: storeName.rustCryptoPrefix,
+    });
   } catch (err) {
     if (!isMismatch(err)) {
       debugLog.error('sync', 'Failed to initialize crypto', { error: err });
       throw err;
     }
     log.warn('initClient: mismatch on initRustCrypto — wiping and retrying:', err);
-    debugLog.warn('sync', 'Crypto init mismatch - wiping stores and retrying', { error: err });
+    debugLog.warn('sync', 'Crypto init mismatch - wiping stores and retrying', {
+      error: err,
+    });
     mx.stopClient();
     await wipeAllStores();
     mx = await buildClient(session);
-    await mx.initRustCrypto({ cryptoDatabasePrefix: storeName.rustCryptoPrefix });
+    await mx.initRustCrypto({
+      cryptoDatabasePrefix: storeName.rustCryptoPrefix,
+    });
   }
 
   mx.setMaxListeners(50);
@@ -375,6 +430,8 @@ export type StartClientConfig = {
   baseUrl?: string;
   slidingSync?: SlidingSyncConfig;
   sessionSlidingSyncOptIn?: boolean;
+  pollTimeoutMs?: number;
+  timelineLimit?: number;
 };
 
 export type ClientSyncDiagnostics = SyncTransportMeta & {
@@ -415,7 +472,12 @@ export const startClient = async (mx: MatrixClient, config?: StartClientConfig):
     hasProxy: hasSlidingProxy,
   });
 
-  const startClassicSync = async (fallbackFromSliding: boolean, reason: SyncTransportReason) => {
+  const CLASSIC_SYNC_STARTUP_TIMEOUT_MS = 45_000;
+
+  const startClassicSync = async (
+    fallbackFromSliding: boolean,
+    reason: SyncTransportReason
+  ): Promise<void> => {
     syncTransportByClient.set(mx, {
       transport: 'classic',
       slidingConfigured: slidingEnabledOnServer,
@@ -426,13 +488,52 @@ export const startClient = async (mx: MatrixClient, config?: StartClientConfig):
       reason,
     });
     Sentry.metrics.count('sable.sync.transport', 1, {
-      attributes: { transport: 'classic', reason, fallback: String(fallbackFromSliding) },
+      attributes: {
+        transport: 'classic',
+        reason,
+        fallback: String(fallbackFromSliding),
+      },
     });
-    await mx.startClient({
-      lazyLoadMembers: true,
-      pollTimeout: FAST_SYNC_POLL_TIMEOUT_MS,
-      threadSupport: true,
+
+    const startupTimeout = new Promise<void>((resolve) => {
+      window.setTimeout(() => {
+        debugLog.warn('sync', 'Classic sync startup timed out', {
+          userId: mx.getUserId(),
+          timeoutMs: CLASSIC_SYNC_STARTUP_TIMEOUT_MS,
+        });
+        resolve();
+      }, CLASSIC_SYNC_STARTUP_TIMEOUT_MS);
     });
+
+    const effectivePollTimeout = config?.pollTimeoutMs ?? FAST_SYNC_POLL_TIMEOUT_MS;
+    const effectiveTimelineLimit = config?.timelineLimit ?? 10;
+
+    const classicFilter = new Filter(mx.getUserId() ?? undefined);
+    classicFilter.setTimelineLimit(effectiveTimelineLimit);
+    // Ensure lazy loading stays on (carried by buildDefaultFilter but explicit here
+    // since we replace the filter entirely rather than merging).
+    const filterDefinition = classicFilter.getDefinition();
+    if (filterDefinition.room) {
+      filterDefinition.room.timeline = filterDefinition.room.timeline ?? {};
+      (filterDefinition.room.timeline as { lazy_load_members?: boolean }).lazy_load_members = true;
+    }
+
+    installStartupFetchRoomEventPatch(mx);
+
+    let syncStarted: Promise<void>;
+    try {
+      syncStarted = mx.startClient({
+        lazyLoadMembers: true,
+        pollTimeout: effectivePollTimeout,
+        threadSupport: true,
+        filter: classicFilter,
+      });
+    } catch (syncErr) {
+      fetchRoomEventStartupCleanupByClient.get(mx)?.();
+      throw syncErr;
+    }
+
+    await Promise.race([syncStarted, startupTimeout]);
     // Attach an ongoing classic-sync observer — equivalent to SlidingSyncManager's
     // onLifecycle listener. Tracks state transitions, initial-sync timing, and errors.
     let classicSyncCount = 0;
@@ -584,16 +685,22 @@ export const startClient = async (mx: MatrixClient, config?: StartClientConfig):
     reason: 'sliding_active',
   });
   Sentry.metrics.count('sable.sync.transport', 1, {
-    attributes: { transport: 'sliding', reason: 'sliding_active', fallback: 'false' },
+    attributes: {
+      transport: 'sliding',
+      reason: 'sliding_active',
+      fallback: 'false',
+    },
   });
 
   try {
+    installStartupFetchRoomEventPatch(mx);
     await mx.startClient({
       lazyLoadMembers: true,
       slidingSync: manager.slidingSync,
       threadSupport: true,
     });
   } catch (err) {
+    fetchRoomEventStartupCleanupByClient.get(mx)?.();
     debugLog.error('network', 'Failed to start client with sliding sync', {
       error: err instanceof Error ? err.message : String(err),
       userId: mx.getUserId(),
@@ -608,6 +715,7 @@ export const startClient = async (mx: MatrixClient, config?: StartClientConfig):
 export const stopClient = (mx: MatrixClient): void => {
   log.log('stopClient', mx.getUserId());
   debugLog.info('sync', 'Stopping client', { userId: mx.getUserId() });
+  fetchRoomEventStartupCleanupByClient.get(mx)?.();
   disposeSlidingSync(mx);
   const classicSyncListener = classicSyncObserverByClient.get(mx);
   if (classicSyncListener) {
@@ -649,7 +757,10 @@ export const getClientSyncDiagnostics = (mx: MatrixClient): ClientSyncDiagnostic
  * so the correct Jotai Provider store is used.
  */
 export const logoutClient = async (mx: MatrixClient, session?: Session) => {
-  log.log('logoutClient', { userId: mx.getUserId(), sessionUserId: session?.userId });
+  log.log('logoutClient', {
+    userId: mx.getUserId(),
+    sessionUserId: session?.userId,
+  });
   debugLog.info('general', 'Logging out client', { userId: mx.getUserId() });
   pushSessionToSW();
   stopClient(mx);

src/types/matrix-sdk.ts

@@ -12,7 +12,7 @@ export * from 'matrix-js-sdk/lib/sliding-sync';
 export * from 'matrix-js-sdk/lib/sync-accumulator';
 export * from 'matrix-js-sdk/lib/scheduler';
 export * from 'matrix-js-sdk/lib/store/memory';
-export { createClient } from 'matrix-js-sdk/lib/matrix';
+export { createClient, Filter } from 'matrix-js-sdk/lib/matrix';
 
 export * from 'matrix-js-sdk/lib/models/event';
 export * from 'matrix-js-sdk/lib/models/room';