From 949e2baa487e96a832d31245bf573e0cca0e6b67 Mon Sep 17 00:00:00 2001 From: Christian Fehmer Date: Thu, 13 Nov 2025 15:31:55 +0100 Subject: [PATCH 1/4] perf: optimize friend queries (@fehmer) (#7080) Combine two queries (first get all friend UIDs, then call leaderboard) into one query to reduce db roundtrips. Use the same approach for the friends list in user dal. Note: when updating mongodb to 6+ we could use unionWith in case we don't need the metadata (lb use-case) --- .../__integration__/dal/connections.spec.ts | 89 ++++++ .../dal/leaderboards.isolated.spec.ts | 60 +++- backend/__tests__/__testData__/connections.ts | 7 +- .../api/controllers/leaderboard.spec.ts | 16 +- backend/src/api/controllers/leaderboard.ts | 32 +- backend/src/dal/connections.ts | 127 +++++++- backend/src/dal/leaderboards.ts | 119 ++++---- backend/src/dal/user.ts | 278 ++++++------------ 8 files changed, 446 insertions(+), 282 deletions(-) diff --git a/backend/__tests__/__integration__/dal/connections.spec.ts b/backend/__tests__/__integration__/dal/connections.spec.ts index 7aa62d99cb47..b7b9af47c325 100644 --- a/backend/__tests__/__integration__/dal/connections.spec.ts +++ b/backend/__tests__/__integration__/dal/connections.spec.ts @@ -11,6 +11,7 @@ import { ObjectId } from "mongodb"; import * as ConnectionsDal from "../../../src/dal/connections"; import { createConnection } from "../../__testData__/connections"; +import { createUser } from "../../__testData__/users"; describe("ConnectionsDal", () => { beforeAll(async () => { @@ -401,4 +402,92 @@ describe("ConnectionsDal", () => { ]); }); }); + + describe("aggregateWithAcceptedConnections", () => { + it("should return friend uids", async () => { + //GIVE + const uid = (await createUser()).uid; + const friendOne = await createConnection({ + initiatorUid: uid, + receiverUid: (await createUser()).uid, + status: "accepted", + }); + const friendTwo = await createConnection({ + initiatorUid: (await createUser()).uid, + receiverUid: uid, + status: "accepted", + }); + const friendThree = await createConnection({ + initiatorUid: (await createUser()).uid, + receiverUid: uid, + status: "accepted", + }); + const _pending = await createConnection({ + initiatorUid: uid, + receiverUid: (await createUser()).uid, + status: "pending", + }); + const _blocked = await createConnection({ + initiatorUid: uid, + receiverUid: (await createUser()).uid, + status: "blocked", + }); + const _decoy = await createConnection({ + receiverUid: (await createUser()).uid, + status: "accepted", + }); + + //WHEN + const friendUids = await ConnectionsDal.aggregateWithAcceptedConnections<{ + uid: string; + }>({ collectionName: "users", uid }, [{ $project: { uid: true } }]); + + //THEN + expect(friendUids.flatMap((it) => it.uid).toSorted()).toEqual([ + uid, + friendOne.receiverUid, + friendTwo.initiatorUid, + friendThree.initiatorUid, + ]); + }); + it("should return friend uids and metaData", async () => { + //GIVE + const me = await createUser(); + const friend = await createUser(); + + const connection = await createConnection({ + initiatorUid: me.uid, + receiverUid: friend.uid, + status: "accepted", + }); + + //WHEN + const friendUids = await ConnectionsDal.aggregateWithAcceptedConnections( + { collectionName: "users", uid: me.uid, includeMetaData: true }, + [ + { + $project: { + uid: true, + lastModified: "$connectionMeta.lastModified", + connectionId: "$connectionMeta._id", + }, + }, + ] + ); + + //THEN + expect(friendUids).toEqual([ + { + _id: friend._id, + connectionId: connection._id, + lastModified: connection.lastModified, + uid: friend.uid, + }, + { + _id: me._id, + uid: me.uid, + }, + ]); + }); + }); }); diff --git a/backend/__tests__/__integration__/dal/leaderboards.isolated.spec.ts b/backend/__tests__/__integration__/dal/leaderboards.isolated.spec.ts index 63a75af779e9..5b37a0877755 100644 --- a/backend/__tests__/__integration__/dal/leaderboards.isolated.spec.ts +++ b/backend/__tests__/__integration__/dal/leaderboards.isolated.spec.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, afterEach, vi } from "vitest"; +import { describe, it, expect, vi, afterEach } from "vitest"; import _ from "lodash"; import { ObjectId } from "mongodb"; import * as UserDal from "../../../src/dal/user"; @@ -11,6 +11,7 @@ import * as DB from "../../../src/init/db"; import { LbPersonalBests } from "../../../src/utils/pb"; import { pb } from "../../__testData__/users"; +import { createConnection } from "../../__testData__/connections"; describe("LeaderboardsDal", () => { afterEach(async () => { @@ -307,9 +308,20 @@ describe("LeaderboardsDal", () => { it("should get for friends only", async () => { //GIVEN const rank1 = await createUser(lbBests(pb(90), pb(100, 90, 2))); + const uid = rank1.uid; const _rank2 = await createUser(lbBests(undefined, pb(100, 90, 1))); const _rank3 = await createUser(lbBests(undefined, pb(100, 80, 2))); const rank4 = await createUser(lbBests(undefined, pb(90, 100, 1))); + + //two friends, one is not on the leaderboard + await createConnection({ + initiatorUid: uid, + receiverUid: rank4.uid, + status: "accepted", + }); + + await createConnection({ initiatorUid: uid, status: "accepted" }); + await LeaderboardsDal.update("time", "60", "english"); //WHEN @@ -321,7 +333,7 @@ describe("LeaderboardsDal", () => { 0, 50, false, - [rank1.uid, rank4.uid] + uid )) as LeaderboardsDal.DBLeaderboardEntry[]; //THEN @@ -335,11 +347,23 @@ describe("LeaderboardsDal", () => { it("should get for friends only with page", async () => { //GIVEN const rank1 = await createUser(lbBests(pb(90), pb(105, 90, 2))); + const uid = rank1.uid; const rank2 = await createUser(lbBests(undefined, pb(100, 90, 1))); const _rank3 = await createUser(lbBests(undefined, pb(95, 80, 2))); const rank4 = await createUser(lbBests(undefined, pb(90, 100, 1))); await LeaderboardsDal.update("time", "60", "english"); + await createConnection({ + initiatorUid: uid, + receiverUid: rank2.uid, + status: "accepted", + }); + await createConnection({ + initiatorUid: rank4.uid, + receiverUid: uid, + status: "accepted", + }); + //WHEN const results = (await LeaderboardsDal.get( "time", @@ -348,7 +372,7 @@ describe("LeaderboardsDal", () => { 1, 2, false, - [rank1.uid, rank2.uid, rank4.uid] + uid )) as LeaderboardsDal.DBLeaderboardEntry[]; //THEN @@ -360,6 +384,7 @@ describe("LeaderboardsDal", () => { }); it("should return empty list if no friends", async () => { //GIVEN + const uid = new ObjectId().toHexString(); //WHEN const results = (await LeaderboardsDal.get( @@ -369,7 +394,7 @@ describe("LeaderboardsDal", () => { 1, 2, false, - [] + uid )) as LeaderboardsDal.DBLeaderboardEntry[]; //THEN expect(results).toEqual([]); @@ -378,10 +403,10 @@ describe("LeaderboardsDal", () => { describe("getCount / getRank", () => { it("should get count", async () => { //GIVEN - await createUser(lbBests(undefined, pb(105))); - await createUser(lbBests(undefined, pb(100))); - const me = await createUser(lbBests(undefined, pb(95))); - await createUser(lbBests(undefined, pb(90))); + await createUser(lbBests(undefined, pb(105)), { name: "One" }); + await createUser(lbBests(undefined, pb(100)), { name: "Two" }); + const me = await createUser(lbBests(undefined, pb(95)), { name: "Me" }); + await createUser(lbBests(undefined, pb(90)), { name: "Three" }); await LeaderboardsDal.update("time", "60", "english"); //WHEN / THEN @@ -405,19 +430,26 @@ describe("LeaderboardsDal", () => { await createUser(lbBests(undefined, pb(95))); const friendTwo = await createUser(lbBests(undefined, pb(90))); const me = await createUser(lbBests(undefined, pb(99))); - - console.log("me", me.uid); - await LeaderboardsDal.update("time", "60", "english"); - const friends = [friendOne.uid, friendTwo.uid, me.uid]; + await createConnection({ + initiatorUid: me.uid, + receiverUid: friendOne.uid, + status: "accepted", + }); + + await createConnection({ + initiatorUid: friendTwo.uid, + receiverUid: me.uid, + status: "accepted", + }); //WHEN / THEN - expect(await LeaderboardsDal.getCount("time", "60", "english", friends)) // + expect(await LeaderboardsDal.getCount("time", "60", "english", me.uid)) // .toEqual(3); expect( - await LeaderboardsDal.getRank("time", "60", "english", me.uid, friends) + await LeaderboardsDal.getRank("time", "60", "english", me.uid, true) ) // .toEqual( expect.objectContaining({ diff --git a/backend/__tests__/__testData__/connections.ts b/backend/__tests__/__testData__/connections.ts index acde3ca09940..38a18f614270 100644 --- a/backend/__tests__/__testData__/connections.ts +++ b/backend/__tests__/__testData__/connections.ts @@ -16,9 +16,8 @@ export async function createConnection( }, maxPerUser ); - await ConnectionsDal.getCollection().updateOne( - { _id: result._id }, - { $set: data } - ); + await ConnectionsDal.__testing + .getCollection() + .updateOne({ _id: result._id }, { $set: data }); return { ...result, ...data }; } diff --git a/backend/__tests__/api/controllers/leaderboard.spec.ts b/backend/__tests__/api/controllers/leaderboard.spec.ts index d06b906f33cb..12fd86ea25ba 100644 --- a/backend/__tests__/api/controllers/leaderboard.spec.ts +++ b/backend/__tests__/api/controllers/leaderboard.spec.ts @@ -30,12 +30,10 @@ describe("Loaderboard Controller", () => { describe("get leaderboard", () => { const getLeaderboardMock = vi.spyOn(LeaderboardDal, "get"); const getLeaderboardCountMock = vi.spyOn(LeaderboardDal, "getCount"); - const getFriendsUidsMock = vi.spyOn(ConnectionsDal, "getFriendsUids"); beforeEach(() => { getLeaderboardMock.mockClear(); getLeaderboardCountMock.mockClear(); - getFriendsUidsMock.mockClear(); getLeaderboardCountMock.mockResolvedValue(42); }); @@ -154,7 +152,6 @@ describe("Loaderboard Controller", () => { //GIVEN await enableConnectionsFeature(true); getLeaderboardMock.mockResolvedValue([]); - getFriendsUidsMock.mockResolvedValue(["uidOne", "uidTwo"]); getLeaderboardCountMock.mockResolvedValue(2); //WHEN @@ -180,13 +177,13 @@ describe("Loaderboard Controller", () => { 0, 50, false, - ["uidOne", "uidTwo"] + uid ); expect(getLeaderboardCountMock).toHaveBeenCalledWith( "time", "60", "english", - ["uidOne", "uidTwo"] + uid ); }); @@ -286,11 +283,9 @@ describe("Loaderboard Controller", () => { describe("get rank", () => { const getLeaderboardRankMock = vi.spyOn(LeaderboardDal, "getRank"); - const getFriendsUidsMock = vi.spyOn(ConnectionsDal, "getFriendsUids"); afterEach(() => { getLeaderboardRankMock.mockClear(); - getFriendsUidsMock.mockClear(); }); it("fails withouth authentication", async () => { @@ -335,14 +330,12 @@ describe("Loaderboard Controller", () => { "60", "english", uid, - undefined + false ); }); it("should get for english time 60 friends only", async () => { //GIVEN await enableConnectionsFeature(true); - const friends = ["friendOne", "friendTwo"]; - getFriendsUidsMock.mockResolvedValue(friends); getLeaderboardRankMock.mockResolvedValue({} as any); //WHEN @@ -363,9 +356,8 @@ describe("Loaderboard Controller", () => { "60", "english", uid, - friends + true ); - expect(getFriendsUidsMock).toHaveBeenCalledWith(uid); }); it("should get with ape key", async () => { await acceptApeKeys(true); diff --git a/backend/src/api/controllers/leaderboard.ts b/backend/src/api/controllers/leaderboard.ts index ae7c47a74d16..e754e38babf2 100644 --- a/backend/src/api/controllers/leaderboard.ts +++ b/backend/src/api/controllers/leaderboard.ts @@ -43,11 +43,7 @@ export async function getLeaderboard( throw new MonkeyError(404, "There is no leaderboard for this mode"); } - const friendUids = await getFriendsUids( - uid, - friendsOnly === true, - connectionsConfig - ); + const friendsOnlyUid = getFriendsOnlyUid(uid, friendsOnly, connectionsConfig); const leaderboard = await LeaderboardsDAL.get( mode, @@ -56,7 +52,7 @@ export async function getLeaderboard( page, pageSize, req.ctx.configuration.users.premium.enabled, - friendUids + friendsOnlyUid ); if (leaderboard === false) { @@ -70,7 +66,7 @@ export async function getLeaderboard( mode, mode2, language, - friendUids + friendsOnlyUid ); const normalizedLeaderboard = leaderboard.map((it) => _.omit(it, ["_id"])); @@ -88,18 +84,12 @@ export async function getRankFromLeaderboard( const { uid } = req.ctx.decodedToken; const connectionsConfig = req.ctx.configuration.connections; - const friendUids = await getFriendsUids( - uid, - friendsOnly === true, - connectionsConfig - ); - const data = await LeaderboardsDAL.getRank( mode, mode2, language, uid, - friendUids + getFriendsOnlyUid(uid, friendsOnly, connectionsConfig) !== undefined ); if (data === false) { throw new MonkeyError( @@ -284,3 +274,17 @@ async function getFriendsUids( } return undefined; } + +function getFriendsOnlyUid( + uid: string, + friendsOnly: boolean | undefined, + friendsConfig: Configuration["connections"] +): string | undefined { + if (uid !== "" && friendsOnly === true) { + if (!friendsConfig.enabled) { + throw new MonkeyError(503, "This feature is currently unavailable."); + } + return uid; + } + return undefined; +} diff --git a/backend/src/dal/connections.ts b/backend/src/dal/connections.ts index a2924bd2cae9..75b2eeb0ab94 100644 --- a/backend/src/dal/connections.ts +++ b/backend/src/dal/connections.ts @@ -1,4 +1,4 @@ -import { Collection, Filter, ObjectId } from "mongodb"; +import { Collection, Document, Filter, ObjectId } from "mongodb"; import * as db from "../init/db"; import { Connection, ConnectionStatus } from "@monkeytype/schemas/connections"; import MonkeyError from "../utils/error"; @@ -10,7 +10,7 @@ export type DBConnection = WithObjectId< } >; -export const getCollection = (): Collection => +const getCollection = (): Collection => db.collection("connections"); export async function getConnections(options: { @@ -209,6 +209,125 @@ export async function getFriendsUids(uid: string): Promise { ); } +/** + * aggregate the given `pipeline` on the `collectionName` for each friendUid and the given `uid`. + + * @param pipeline + * @param options + * @returns + */ +export async function aggregateWithAcceptedConnections( + options: { + uid: string; + /** + * target collection + */ + collectionName: string; + /** + * uid field on the collection, defaults to `uid` + */ + uidField?: string; + /** + * add meta data `connectionMeta.lastModified` and *connectionMeta._id` to the document + */ + includeMetaData?: boolean; + }, + pipeline: Document[] +): Promise { + const metaData = options.includeMetaData + ? { + let: { + lastModified: "$lastModified", + connectionId: "$connectionId", + }, + pipeline: [ + { + $addFields: { + "connectionMeta.lastModified": "$$lastModified", + "connectionMeta._id": "$$connectionId", + }, + }, + ], + } + : {}; + const { uid, collectionName, uidField } = options; + const fullPipeline = [ + { + $match: { + status: "accepted", + //uid is friend or initiator + $or: [{ initiatorUid: uid }, { receiverUid: uid }], + }, + }, + { + $project: { + lastModified: true, + uid: { + //pick the other user, not uid + $cond: { + if: { $eq: ["$receiverUid", uid] }, + // oxlint-disable-next-line no-thenable + then: "$initiatorUid", + else: "$receiverUid", + }, + }, + }, + }, + // we want to fetch the data for our uid as well, add it to the list of documents + // workaround for missing unionWith + $documents in mongodb 5.0 + { + $group: { + _id: null, + data: { + $push: { + uid: "$uid", + lastModified: "$lastModified", + connectionId: "$_id", + }, + }, + }, + }, + { + $project: { + data: { + $concatArrays: ["$data", [{ uid }]], + }, + }, + }, + { $unwind: "$data" }, + { $replaceRoot: { newRoot: "$data" } }, + + /* end of workaround, this is the replacement for >= 5.1 + + { $addFields: { connectionId: "$_id" } }, + { $project: { uid: true, lastModified: true, connectionId: true } }, + { + $unionWith: { + pipeline: [{ $documents: [{ uid }] }], + }, + }, + */ + + { + //replace with $unionWith in MongoDB 6 or newer + $lookup: { + from: collectionName, + localField: "uid", + foreignField: uidField ?? "uid", + as: "result", + ...metaData, + }, + }, + + { $match: { result: { $ne: [] } } }, + { $replaceRoot: { newRoot: { $first: "$result" } } }, + ...pipeline, + ]; + + //console.log(JSON.stringify(fullPipeline, null, 4)); + return (await getCollection().aggregate(fullPipeline).toArray()) as T[]; +} + function getKey(initiatorUid: string, receiverUid: string): string { const ids = [initiatorUid, receiverUid]; ids.sort(); @@ -223,3 +342,7 @@ export async function createIndicies(): Promise { //make sure there is only one connection for each initiatorr/receiver await getCollection().createIndex({ key: 1 }, { unique: true }); } + +export const __testing = { + getCollection, +}; diff --git a/backend/src/dal/leaderboards.ts b/backend/src/dal/leaderboards.ts index 8de1b68bb9bf..573d6c9308eb 100644 --- a/backend/src/dal/leaderboards.ts +++ b/backend/src/dal/leaderboards.ts @@ -14,19 +14,25 @@ import { LeaderboardEntry } from "@monkeytype/schemas/leaderboards"; import { omit } from "lodash"; import { DBUser, getUsersCollection } from "./user"; import MonkeyError from "../utils/error"; +import { aggregateWithAcceptedConnections } from "./connections"; export type DBLeaderboardEntry = LeaderboardEntry & { _id: ObjectId; }; +function getCollectionName(key: { + language: string; + mode: string; + mode2: string; +}): string { + return `leaderboards.${key.language}.${key.mode}.${key.mode2}`; +} export const getCollection = (key: { language: string; mode: string; mode2: string; }): Collection => - db.collection( - `leaderboards.${key.language}.${key.mode}.${key.mode2}` - ); + db.collection(getCollectionName(key)); export async function get( mode: string, @@ -35,42 +41,45 @@ export async function get( page: number, pageSize: number, premiumFeaturesEnabled: boolean = false, - userIds?: string[] + uid?: string ): Promise { if (page < 0 || pageSize < 0) { throw new MonkeyError(500, "Invalid page or pageSize"); } - if (userIds?.length === 0) { - return []; - } - const skip = page * pageSize; const limit = pageSize; + let leaderboard: DBLeaderboardEntry[] | false = []; + const pipeline: Document[] = [ { $sort: { rank: 1 } }, { $skip: skip }, { $limit: limit }, ]; - if (userIds !== undefined) { - pipeline.unshift( - { $match: { uid: { $in: userIds } } }, - { - $setWindowFields: { - sortBy: { rank: 1 }, - output: { friendsRank: { $documentNumber: {} } }, - }, - } - ); - } - try { - let leaderboard = (await getCollection({ language, mode, mode2 }) - .aggregate(pipeline) - .toArray()) as DBLeaderboardEntry[]; - + if (uid !== undefined) { + leaderboard = await aggregateWithAcceptedConnections( + { + uid, + collectionName: getCollectionName({ language, mode, mode2 }), + }, + [ + { + $setWindowFields: { + sortBy: { rank: 1 }, + output: { friendsRank: { $documentNumber: {} } }, + }, + }, + ...pipeline, + ] + ); + } else { + leaderboard = await getCollection({ language, mode, mode2 }) + .aggregate(pipeline) + .toArray(); + } if (!premiumFeaturesEnabled) { leaderboard = leaderboard.map((it) => omit(it, "isPremium")); } @@ -92,23 +101,30 @@ export async function getCount( mode: string, mode2: string, language: string, - userIds?: string[] + uid?: string ): Promise { const key = `${language}_${mode}_${mode2}`; - if (userIds === undefined && cachedCounts.has(key)) { + if (uid === undefined && cachedCounts.has(key)) { return cachedCounts.get(key) as number; } else { - const lb = getCollection({ - language, - mode, - mode2, - }); - if (userIds === undefined) { - const count = await lb.estimatedDocumentCount(); + if (uid === undefined) { + const count = await getCollection({ + language, + mode, + mode2, + }).estimatedDocumentCount(); cachedCounts.set(key, count); return count; } else { - return lb.countDocuments({ uid: { $in: userIds } }); + return ( + await aggregateWithAcceptedConnections( + { + collectionName: getCollectionName({ language, mode, mode2 }), + uid, + }, + [{ $project: { _id: true } }] + ) + ).length; } } } @@ -118,32 +134,33 @@ export async function getRank( mode2: string, language: string, uid: string, - userIds?: string[] + friendsOnly: boolean = false ): Promise { try { - if (userIds === undefined) { + if (!friendsOnly) { const entry = await getCollection({ language, mode, mode2 }).findOne({ uid, }); return entry; - } else if (userIds.length === 0) { - return null; } else { - const entry = await getCollection({ language, mode, mode2 }) - .aggregate([ - { $match: { uid: { $in: userIds } } }, + const results = + await aggregateWithAcceptedConnections( { - $setWindowFields: { - sortBy: { rank: 1 }, - output: { friendsRank: { $documentNumber: {} } }, - }, + collectionName: getCollectionName({ language, mode, mode2 }), + uid, }, - { $match: { uid } }, - ]) - .toArray(); - - return entry[0] as DBLeaderboardEntry; + [ + { + $setWindowFields: { + sortBy: { rank: 1 }, + output: { friendsRank: { $documentNumber: {} } }, + }, + }, + { $match: { uid } }, + ] + ); + return results[0] ?? null; } } catch (e) { // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access @@ -164,7 +181,7 @@ export async function update( rank?: number; }> { const key = `lbPersonalBests.${mode}.${mode2}.${language}`; - const lbCollectionName = `leaderboards.${language}.${mode}.${mode2}`; + const lbCollectionName = getCollectionName({ language, mode, mode2 }); const minTimeTyping = (await getCachedConfiguration(true)).leaderboards .minTimeTyping; const lb = db.collection("users").aggregate( diff --git a/backend/src/dal/user.ts b/backend/src/dal/user.ts index 756cd2793dcd..a96bdc67a90e 100644 --- a/backend/src/dal/user.ts +++ b/backend/src/dal/user.ts @@ -34,7 +34,7 @@ import { Result as ResultType } from "@monkeytype/schemas/results"; import { Configuration } from "@monkeytype/schemas/configuration"; import { isToday, isYesterday } from "@monkeytype/util/date-and-time"; import GeorgeQueue from "../queues/george-queue"; -import { getCollection as getConnectionCollection } from "./connections"; +import { aggregateWithAcceptedConnections } from "./connections"; export type DBUserTag = WithObjectId; @@ -1228,210 +1228,118 @@ async function updateUser( } export async function getFriends(uid: string): Promise { - return (await getConnectionCollection() - .aggregate([ - { - $match: { - //uid is friend or initiator - $and: [ - { - $or: [{ initiatorUid: uid }, { receiverUid: uid }], - status: "accepted", - }, - ], - }, - }, + return await aggregateWithAcceptedConnections( + { + uid, + collectionName: "users", + includeMetaData: true, + }, + [ { $project: { - receiverUid: true, - initiatorUid: true, - lastModified: true, + _id: false, + uid: true, + connectionId: "$connectionMeta._id", + lastModified: "$connectionMeta.lastModified", + name: true, + discordId: true, + discordAvatar: true, + startedTests: true, + completedTests: true, + timeTyping: true, + xp: true, + "streak.length": true, + "streak.maxLength": true, + personalBests: true, + "inventory.badges": true, + "premium.expirationTimestamp": true, + banned: 1, + lbOptOut: 1, }, }, { $addFields: { - //pick the other user, not uid - uid: { - $cond: { - if: { $eq: ["$receiverUid", uid] }, - // oxlint-disable-next-line no-thenable - then: "$initiatorUid", - else: "$receiverUid", - }, - }, - }, - }, - // we want to fetch the data for our uid as well, add it to the list of documents - // workaround for missing unionWith + $documents in mongodb 5.0 - { - $group: { - _id: null, - data: { - $push: { - uid: "$uid", - lastModified: "$lastModified", - connectionId: "$_id", + top15: { + $reduce: { + //find highest wpm from time 15 PBs + input: "$personalBests.time.15", + initialValue: {}, + in: { + $cond: [ + { $gte: ["$$this.wpm", "$$value.wpm"] }, + "$$this", + "$$value", + ], + }, }, }, - }, - }, - { - $project: { - data: { - $concatArrays: ["$data", [{ uid }]], - }, - }, - }, - { - $unwind: "$data", - }, - - /* end of workaround, this is the replacement for >= 5.1 - - { $addFields: { connectionId: "$_id" } }, - { $project: { uid: true, lastModified: true, connectionId: true } }, - { - $unionWith: { - pipeline: [{ $documents: [{ uid }] }], - }, - }, - */ - - { - $lookup: { - /* query users to get the friend data */ - from: "users", - localField: "data.uid", //just uid if we remove the workaround above - foreignField: "uid", - as: "result", - let: { - lastModified: "$data.lastModified", //just $lastModified if we remove the workaround above - connectionId: "$data.connectionId", //just $connectionId if we remove the workaround above - }, - pipeline: [ - { - $project: { - _id: false, - uid: true, - connectionId: true, - name: true, - discordId: true, - discordAvatar: true, - startedTests: true, - completedTests: true, - timeTyping: true, - xp: true, - "streak.length": true, - "streak.maxLength": true, - personalBests: true, - "inventory.badges": true, - "premium.expirationTimestamp": true, - banned: 1, - lbOptOut: 1, + top60: { + $reduce: { + //find highest wpm from time 60 PBs + input: "$personalBests.time.60", + initialValue: {}, + in: { + $cond: [ + { $gte: ["$$this.wpm", "$$value.wpm"] }, + "$$this", + "$$value", + ], }, }, - { - $addFields: { - lastModified: "$$lastModified", - connectionId: "$$connectionId", - top15: { - $reduce: { - //find highest wpm from time 15 PBs - input: "$personalBests.time.15", - initialValue: {}, - in: { - $cond: [ - { $gte: ["$$this.wpm", "$$value.wpm"] }, - "$$this", - "$$value", - ], - }, - }, - }, - top60: { - $reduce: { - //find highest wpm from time 60 PBs - input: "$personalBests.time.60", - initialValue: {}, - in: { - $cond: [ - { $gte: ["$$this.wpm", "$$value.wpm"] }, - "$$this", - "$$value", - ], - }, - }, - }, - badgeId: { - $ifNull: [ - { - $first: { - $map: { - input: { - $filter: { - input: "$inventory.badges", - as: "badge", - cond: { $eq: ["$$badge.selected", true] }, - }, - }, - as: "selectedBadge", - in: "$$selectedBadge.id", - }, + }, + badgeId: { + $ifNull: [ + { + $first: { + $map: { + input: { + $filter: { + input: "$inventory.badges", + as: "badge", + cond: { $eq: ["$$badge.selected", true] }, }, }, - "$$REMOVE", - ], - }, - isPremium: { - $cond: { - if: { - $or: [ - { $eq: ["$premium.expirationTimestamp", -1] }, - { - $gt: [ - "$premium.expirationTimestamp", - { $toLong: "$$NOW" }, - ], - }, - ], - }, - // oxlint-disable-next-line no-thenable - then: true, - else: "$$REMOVE", + as: "selectedBadge", + in: "$$selectedBadge.id", }, }, }, - }, - { - $addFields: { - //remove nulls - top15: { $ifNull: ["$top15", "$$REMOVE"] }, - top60: { $ifNull: ["$top60", "$$REMOVE"] }, - badgeId: { $ifNull: ["$badgeId", "$$REMOVE"] }, - lastModified: "$lastModified", - }, - }, - { - $project: { - personalBests: false, - inventory: false, - premium: false, + "$$REMOVE", + ], + }, + isPremium: { + $cond: { + if: { + $or: [ + { $eq: ["$premium.expirationTimestamp", -1] }, + { + $gt: ["$premium.expirationTimestamp", { $toLong: "$$NOW" }], + }, + ], }, + // oxlint-disable-next-line no-thenable + then: true, + else: "$$REMOVE", }, - ], + }, }, }, { - $replaceRoot: { - newRoot: { - $cond: [ - { $gt: [{ $size: "$result" }, 0] }, - { $first: "$result" }, - {}, // empty document fallback, this can happen if the user is not present - ], - }, + $addFields: { + //remove nulls + top15: { $ifNull: ["$top15", "$$REMOVE"] }, + top60: { $ifNull: ["$top60", "$$REMOVE"] }, + badgeId: { $ifNull: ["$badgeId", "$$REMOVE"] }, + lastModified: "$lastModified", }, }, - ]) - .toArray()) as DBFriend[]; + { + $project: { + personalBests: false, + inventory: false, + premium: false, + }, + }, + ] + ); } From bb6e0d9824e126defd15c13df53761c73d4297a5 Mon Sep 17 00:00:00 2001 From: Sameer Singh <110624457+SameerJS6@users.noreply.github.com> Date: Thu, 13 Nov 2025 20:04:51 +0530 Subject: [PATCH 2/4] feat(theme): add Vesper Light (@SameerJS6) (#7040) ### Description Adds Vesper Light and Oscura themes. Oscura by [Fey](https://github.com/narative/oscura). ### Checks - [ ] Adding quotes? - [ ] Make sure to include translations for the quotes in the description (or another comment) so we can verify their content. - [ ] Adding a language? - Make sure to follow the [languages documentation](https://github.com/monkeytypegame/monkeytype/blob/master/docs/LANGUAGES.md) - [ ] Add language to `packages/schemas/src/languages.ts` - [ ] Add language to exactly one group in `frontend/src/ts/constants/languages.ts` - [ ] Add language json file to `frontend/static/languages` - [x] Adding a theme? - Make sure to follow the [themes documentation](https://github.com/monkeytypegame/monkeytype/blob/master/docs/THEMES.md) - [x] Add theme to `packages/schemas/src/themes.ts` - [x] Add theme to `frontend/src/ts/constants/themes.ts` - [x] Add theme css file to `frontend/static/themes` - [x] Add some screenshot of the theme, especially with different test settings (colorful, flip colors) to your pull request - [ ] Adding a layout? - [ ] Make sure to follow the [layouts documentation](https://github.com/monkeytypegame/monkeytype/blob/master/docs/LAYOUTS.md) - [ ] Add layout to `packages/schemas/src/layouts.ts` - [ ] Add layout json file to `frontend/static/layouts` - [ ] Adding a font? - Make sure to follow the [themes documentation](https://github.com/monkeytypegame/monkeytype/blob/master/docs/FONTS.md) - [ ] Add font file to `frontend/static/webfonts` - [ ] Add font to `packages/schemas/src/fonts.ts` - [ ] Add font to `frontend/src/ts/constants/fonts.ts` - [x] Check if any open issues are related to this PR; if so, be sure to tag them below. - [x] Make sure the PR title follows the Conventional Commits standard. (https://www.conventionalcommits.org for more info) - [x] Make sure to include your GitHub username prefixed with @ inside parentheses at the end of the PR title. Closes # ## Preview ### Vesper Light #### Home Route vesper-light #### Settings Route 80_1x_shots_so #### Result Route 908_1x_shots_so ### Oscura #### Home Route 789_1x_shots_so #### Settings Route 194_1x_shots_so #### Result Route 95_1x_shots_so ### Toasts Theme For Vesper Light & Oscura 585_1x_shots_so --- frontend/src/ts/constants/themes.ts | 6 +++ frontend/static/themes/vesper.css | 4 +- frontend/static/themes/vesper_light.css | 56 +++++++++++++++++++++++++ packages/schemas/src/themes.ts | 1 + 4 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 frontend/static/themes/vesper_light.css diff --git a/frontend/src/ts/constants/themes.ts b/frontend/src/ts/constants/themes.ts index dd74af172235..b7147af8068c 100644 --- a/frontend/src/ts/constants/themes.ts +++ b/frontend/src/ts/constants/themes.ts @@ -1114,6 +1114,12 @@ export const themes: Record> = { subColor: "#5b578e", textColor: "#f4e0c9", }, + vesper_light: { + bgColor: "#ffffff", + mainColor: "#fb7100", + subColor: "#067a6e", + textColor: "#000000", + }, pale_nimbus: { bgColor: "#433e4c", mainColor: "#94ffc2", diff --git a/frontend/static/themes/vesper.css b/frontend/static/themes/vesper.css index 2e643bc5ef9c..c13b9a85e7fa 100644 --- a/frontend/static/themes/vesper.css +++ b/frontend/static/themes/vesper.css @@ -7,8 +7,8 @@ --text-color: #ffffff; --error-color: #ff8080; --error-extra-color: #b25959; - --colorful-error-color: #99ffe4; - --colorful-error-extra-color: #99ffe4; + --colorful-error-color: #ff8080; + --colorful-error-extra-color: #b25959; } /* Background & Text Colors */ diff --git a/frontend/static/themes/vesper_light.css b/frontend/static/themes/vesper_light.css new file mode 100644 index 000000000000..00e227249399 --- /dev/null +++ b/frontend/static/themes/vesper_light.css @@ -0,0 +1,56 @@ +:root { + --bg-color: #ffffff; + --main-color: #fb7100; + --caret-color: #067a6e; + --sub-color: #a0a0a0; + --sub-alt-color: #fff8f4; + --text-color: #000000; + --error-color: #ed2839; + --error-extra-color: #ff6c72; + --colorful-error-color: #ed2839; + --colorful-error-extra-color: #ff6c72; +} + +#testConfig .spacer { + background: #efefef; +} + +/* Background & Text Colors */ +#notificationCenter .notif.bad { + --notif-border-color: #ed2939; + --notif-background-color: #ed2839; +} + +#notificationCenter .notif.good { + --notif-border-color: #067a6e; + --notif-background-color: #067a6e; +} + +#notificationCenter .notif.notice { + --notif-border-color: #1e90ff; + --notif-background-color: #1e90ff; +} + +/* Hover Styles */ +#notificationCenter .notif:hover.good { + --notif-background-color: #00a693; +} +#notificationCenter .notif:hover.bad { + --notif-background-color: #ff6c72; +} +#notificationCenter .notif:hover.notice { + --notif-background-color: #4ec5ff; +} + +/* Content Colors */ +#notificationCenter .notif .message { + color: var(--bg-color); +} + +#notificationCenter .notif .message .title { + color: var(--bg-color); +} + +#notificationCenter .notif .icon { + color: var(--bg-color); +} diff --git a/packages/schemas/src/themes.ts b/packages/schemas/src/themes.ts index e31f7724bcd0..ac5463a465e8 100644 --- a/packages/schemas/src/themes.ts +++ b/packages/schemas/src/themes.ts @@ -181,6 +181,7 @@ export const ThemeNameSchema = z.enum( "tron_orange", "vaporwave", "vesper", + "vesper_light", "viridescent", "voc", "vscode", From 05afcc51f9d47a87819144669f464093befe47d3 Mon Sep 17 00:00:00 2001 From: Christian Fehmer Date: Thu, 13 Nov 2025 15:40:10 +0100 Subject: [PATCH 3/4] impr(leaderboard): fix icons on friend xp leaderboards (@fehmer) (#7106) --- frontend/src/html/pages/leaderboards.html | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/frontend/src/html/pages/leaderboards.html b/frontend/src/html/pages/leaderboards.html index 501c423798a4..9f0453999b83 100644 --- a/frontend/src/html/pages/leaderboards.html +++ b/frontend/src/html/pages/leaderboards.html @@ -82,8 +82,21 @@ - - # + + + + + + + + # + + + name xp gained time typed From 0e4b9c4687d5901995e06f163109311d7b3e4824 Mon Sep 17 00:00:00 2001 From: Christian Fehmer Date: Thu, 13 Nov 2025 17:12:37 +0100 Subject: [PATCH 4/4] impr: handle backend unavailable in remote validations (@fehmer) (#7105) --- frontend/src/ts/elements/input-validation.ts | 4 ++- frontend/src/ts/modals/edit-tag.ts | 3 +- frontend/src/ts/modals/google-sign-up.ts | 16 +++------- frontend/src/ts/modals/simple-modals.ts | 16 +++------- frontend/src/ts/pages/friends.ts | 16 +++------- frontend/src/ts/pages/login.ts | 16 +++------- frontend/src/ts/utils/remote-validation.ts | 32 ++++++++++++++++++++ frontend/src/ts/utils/simple-modal.ts | 6 +++- 8 files changed, 62 insertions(+), 47 deletions(-) create mode 100644 frontend/src/ts/utils/remote-validation.ts diff --git a/frontend/src/ts/elements/input-validation.ts b/frontend/src/ts/elements/input-validation.ts index a1971bd622f8..0ad5554b21b8 100644 --- a/frontend/src/ts/elements/input-validation.ts +++ b/frontend/src/ts/elements/input-validation.ts @@ -14,6 +14,8 @@ export type ValidationResult = { errorMessage?: string; }; +export type IsValidResponse = true | string | { warning: string }; + export type Validation = { /** * Zod schema to validate the input value against. @@ -28,7 +30,7 @@ export type Validation = { * @param thisPopup the current modal * @returns true if the `value` is valid, an errorMessage as string if it is invalid. */ - isValid?: (value: T) => Promise; + isValid?: (value: T) => Promise; /** custom debounce delay for `isValid` call. defaults to 100 */ debounceDelay?: number; diff --git a/frontend/src/ts/modals/edit-tag.ts b/frontend/src/ts/modals/edit-tag.ts index 701169818607..eacd882fd775 100644 --- a/frontend/src/ts/modals/edit-tag.ts +++ b/frontend/src/ts/modals/edit-tag.ts @@ -1,12 +1,13 @@ import Ape from "../ape"; import * as DB from "../db"; +import { IsValidResponse } from "../elements/input-validation"; import * as Settings from "../pages/settings"; import AnimatedModal, { ShowOptions } from "../utils/animated-modal"; import { SimpleModal, TextInput } from "../utils/simple-modal"; import { TagNameSchema } from "@monkeytype/schemas/users"; const cleanTagName = (tagName: string): string => tagName.replaceAll(" ", "_"); -const tagNameValidation = async (tagName: string): Promise => { +const tagNameValidation = async (tagName: string): Promise => { const validationResult = TagNameSchema.safeParse(cleanTagName(tagName)); if (validationResult.success) return true; return validationResult.error.errors.map((err) => err.message).join(", "); diff --git a/frontend/src/ts/modals/google-sign-up.ts b/frontend/src/ts/modals/google-sign-up.ts index 278a2e10d8d9..fc452a05eff3 100644 --- a/frontend/src/ts/modals/google-sign-up.ts +++ b/frontend/src/ts/modals/google-sign-up.ts @@ -16,6 +16,7 @@ import AnimatedModal from "../utils/animated-modal"; import { resetIgnoreAuthCallback } from "../firebase"; import { validateWithIndicator } from "../elements/input-validation"; import { UserNameSchema } from "@monkeytype/schemas/users"; +import { remoteValidation } from "../utils/remote-validation"; let signedInUser: UserCredential | undefined = undefined; @@ -154,17 +155,10 @@ function disableInput(): void { validateWithIndicator(nameInputEl, { schema: UserNameSchema, - isValid: async (name: string) => { - const checkNameResponse = await Ape.users.getNameAvailability({ - params: { name: name }, - }); - - return ( - (checkNameResponse.status === 200 && - checkNameResponse.body.data.available) || - "Name not available" - ); - }, + isValid: remoteValidation( + async (name) => Ape.users.getNameAvailability({ params: { name } }), + { check: (data) => data.available || "Name not available" } + ), debounceDelay: 1000, callback: (result) => { if (result.status === "success") { diff --git a/frontend/src/ts/modals/simple-modals.ts b/frontend/src/ts/modals/simple-modals.ts index 019dd2e4c318..74b74103e2f4 100644 --- a/frontend/src/ts/modals/simple-modals.ts +++ b/frontend/src/ts/modals/simple-modals.ts @@ -45,6 +45,7 @@ import { import { goToPage } from "../pages/leaderboards"; import FileStorage from "../utils/file-storage"; import { z } from "zod"; +import { remoteValidation } from "../utils/remote-validation"; type PopupKey = | "updateEmail" @@ -479,17 +480,10 @@ list.updateName = new SimpleModal({ initVal: "", validation: { schema: UserNameSchema, - isValid: async (newName: string) => { - const checkNameResponse = await Ape.users.getNameAvailability({ - params: { name: newName }, - }); - - return ( - (checkNameResponse.status === 200 && - checkNameResponse.body.data.available) || - "Name not available" - ); - }, + isValid: remoteValidation( + async (name) => Ape.users.getNameAvailability({ params: { name } }), + { check: (data) => data.available || "Name not available" } + ), debounceDelay: 1000, }, }, diff --git a/frontend/src/ts/pages/friends.ts b/frontend/src/ts/pages/friends.ts index eb1f30aafb4a..0f7152e13022 100644 --- a/frontend/src/ts/pages/friends.ts +++ b/frontend/src/ts/pages/friends.ts @@ -29,6 +29,7 @@ import { Connection } from "@monkeytype/schemas/connections"; import { Friend, UserNameSchema } from "@monkeytype/schemas/users"; import * as Loader from "../elements/loader"; import { LocalStorageWithSchema } from "../utils/local-storage-with-schema"; +import { remoteValidation } from "../utils/remote-validation"; const pageElement = $(".page.pageFriends"); @@ -75,17 +76,10 @@ const addFriendModal = new SimpleModal({ initVal: "", validation: { schema: UserNameSchema, - isValid: async (name: string) => { - const checkNameResponse = await Ape.users.getNameAvailability({ - params: { name: name }, - }); - - return ( - (checkNameResponse.status === 200 && - !checkNameResponse.body.data.available) || - "Unknown user" - ); - }, + isValid: remoteValidation( + async (name) => Ape.users.getNameAvailability({ params: { name } }), + { check: (data) => !data.available || "Unknown user" } + ), debounceDelay: 1000, }, }, diff --git a/frontend/src/ts/pages/login.ts b/frontend/src/ts/pages/login.ts index b9a83b2d45db..7111ec3fa6ab 100644 --- a/frontend/src/ts/pages/login.ts +++ b/frontend/src/ts/pages/login.ts @@ -10,6 +10,7 @@ import { import { validateWithIndicator } from "../elements/input-validation"; import { isDevEnvironment } from "../utils/misc"; import { z } from "zod"; +import { remoteValidation } from "../utils/remote-validation"; let registerForm: { name?: string; @@ -73,17 +74,10 @@ const nameInputEl = document.querySelector( ) as HTMLInputElement; validateWithIndicator(nameInputEl, { schema: UserNameSchema, - isValid: async (name: string) => { - const checkNameResponse = await Ape.users.getNameAvailability({ - params: { name: name }, - }); - - return ( - (checkNameResponse.status === 200 && - checkNameResponse.body.data.available) || - "Name not available" - ); - }, + isValid: remoteValidation( + async (name) => Ape.users.getNameAvailability({ params: { name } }), + { check: (data) => data.available || "Name not available" } + ), debounceDelay: 1000, callback: (result) => { registerForm.name = diff --git a/frontend/src/ts/utils/remote-validation.ts b/frontend/src/ts/utils/remote-validation.ts new file mode 100644 index 000000000000..0b90198fdaaa --- /dev/null +++ b/frontend/src/ts/utils/remote-validation.ts @@ -0,0 +1,32 @@ +import { IsValidResponse } from "../elements/input-validation"; + +type IsValidResonseOrFunction = + | ((message: string) => IsValidResponse) + | IsValidResponse; +export function remoteValidation( + call: ( + val: V + ) => Promise<{ status: number; body: { data?: T; message: string } }>, + options?: { + check?: (data: T) => IsValidResponse; + on4xx?: IsValidResonseOrFunction; + on5xx?: IsValidResonseOrFunction; + } +): (val: V) => Promise { + return async (val) => { + const result = await call(val); + if (result.status <= 299) { + return options?.check?.(result.body.data as T) ?? true; + } + + let handler: IsValidResonseOrFunction | undefined; + if (result.status <= 499) { + handler = options?.on4xx ?? ((message) => message); + } else { + handler = options?.on5xx ?? "Server unavailable. Please try again later."; + } + + if (typeof handler === "function") return handler(result.body.message); + return handler; + }; +} diff --git a/frontend/src/ts/utils/simple-modal.ts b/frontend/src/ts/utils/simple-modal.ts index f5b62be0491a..a311fee467e1 100644 --- a/frontend/src/ts/utils/simple-modal.ts +++ b/frontend/src/ts/utils/simple-modal.ts @@ -5,6 +5,7 @@ import * as Loader from "../elements/loader"; import * as Notifications from "../elements/notifications"; import * as ConnectionState from "../states/connection"; import { + IsValidResponse, Validation, ValidationOptions, ValidationResult, @@ -33,7 +34,10 @@ type CommonInput = { * @param thisPopup the current modal * @returns true if the `value` is valid, an errorMessage as string if it is invalid. */ - isValid?: (value: string, thisPopup: SimpleModal) => Promise; + isValid?: ( + value: string, + thisPopup: SimpleModal + ) => Promise; }; };