diff --git a/src/api/index.ts b/src/api/index.ts index 5cd4553d..458c8cdb 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -24,7 +24,6 @@ import { } from "fastify-zod-openapi"; import { type ZodOpenApiVersion } from "zod-openapi"; import { withTags } from "./components/index.js"; -import RedisModule from "ioredis"; /** BEGIN EXTERNAL PLUGINS */ import fastifyIp from "fastify-ip"; @@ -61,6 +60,7 @@ import mobileWalletV2Route from "./routes/v2/mobileWallet.js"; import membershipV2Plugin from "./routes/v2/membership.js"; import { docsHtml, securitySchemes } from "./docs.js"; import syncIdentityPlugin from "./routes/syncIdentity.js"; +import { createRedisModule } from "./redis.js"; /** END ROUTES */ export const instanceId = randomUUID(); @@ -309,7 +309,11 @@ Otherwise, email [infra@acm.illinois.edu](mailto:infra@acm.illinois.edu) for sup ) as SecretConfig; }; await app.refreshSecretConfig(); - app.redisClient = new RedisModule.default(app.secretConfig.redis_url); + app.redisClient = await createRedisModule( + app.secretConfig.redis_url, + app.secretConfig.fallback_redis_url, + app.log, + ); } if (isRunningInLambda) { await app.register(fastifyIp.default, { diff --git a/src/api/redis.ts b/src/api/redis.ts new file mode 100644 index 00000000..f8134d12 --- /dev/null +++ b/src/api/redis.ts @@ -0,0 +1,76 @@ +import RedisModule from "ioredis"; +import { ValidLoggers } from "./types.js"; + +type RedisMode = "read-write" | "read-only" | "down"; + +async function checkRedisMode(url: string): Promise { + let testModule: RedisModule.default | null = null; + try { + testModule = new RedisModule.default(url); + + // Test connectivity + await testModule.ping(); + + // Test write capability + const testKey = `health-check:${Date.now()}`; + try { + await testModule.set(testKey, "test", "EX", 1); + return "read-write"; + } catch (writeError) { + return "read-only"; + } + } catch (error) { + return "down"; + } finally { + if (testModule) { + await testModule.quit().catch(() => {}); + } + } +} + +export async function createRedisModule( + primaryUrl: string, + fallbackUrl: string, + logger: ValidLoggers, +) { + const primaryMode = await checkRedisMode(primaryUrl); + if (Math.random() < 0.01) { + // Upstash will complain if we never hit the instance + // Fire a hit every so often on the fallback + // Don't block on it + // Given we create a client once ever hour, we should hit the node at least once every 4 days at least + checkRedisMode(fallbackUrl); + } + if (primaryMode === "read-write") { + logger.info("Using primary Redis in read-write mode"); + return new RedisModule.default(primaryUrl); + } + + const fallbackMode = await checkRedisMode(fallbackUrl); + + if (fallbackMode === "read-write") { + logger.warn( + `Primary Redis is ${primaryMode}, using fallback in read-write mode`, + ); + return new RedisModule.default(fallbackUrl); + } + + if (primaryMode === "read-only") { + logger.warn( + "Both Redis instances are read-only. Using primary in read-only mode", + ); + return new RedisModule.default(primaryUrl); + } + + if (fallbackMode === "read-only") { + logger.error( + "Primary Redis is down and Fallback Redis is down, using primary anyway", + ); + return new RedisModule.default(primaryUrl); + } + + logger.error( + "Both primary and fallback Redis instances are down. Creating client on primary anyway.", + ); + return new RedisModule.default(primaryUrl); +} diff --git a/src/api/sqs/handlers/createOrgGithubTeam.ts b/src/api/sqs/handlers/createOrgGithubTeam.ts index fdfaef1c..d08f6d11 100644 --- a/src/api/sqs/handlers/createOrgGithubTeam.ts +++ b/src/api/sqs/handlers/createOrgGithubTeam.ts @@ -20,6 +20,7 @@ import { Modules } from "common/modules.js"; import { retryDynamoTransactionWithBackoff } from "api/utils.js"; import { SKIP_EXTERNAL_ORG_LEAD_UPDATE } from "common/overrides.js"; import { getOrgByName } from "@acm-uiuc/js-shared"; +import { createRedisModule } from "api/redis.js"; export const createOrgGithubTeamHandler: SQSHandlerFunction< AvailableSQSFunctions.CreateOrgGithubTeam @@ -28,7 +29,11 @@ export const createOrgGithubTeamHandler: SQSHandlerFunction< logger, commonConfig: { region: genericConfig.AwsRegion }, }); - const redisClient = new RedisModule.default(secretConfig.redis_url); + const redisClient = await createRedisModule( + secretConfig.redis_url, + secretConfig.fallback_redis_url, + logger, + ); try { const { orgName, githubTeamName, githubTeamDescription } = payload; const orgImmutableId = getOrgByName(orgName)!.id; diff --git a/src/api/sqs/handlers/provisionNewMember.ts b/src/api/sqs/handlers/provisionNewMember.ts index ff4c734a..cb0c6491 100644 --- a/src/api/sqs/handlers/provisionNewMember.ts +++ b/src/api/sqs/handlers/provisionNewMember.ts @@ -13,6 +13,7 @@ import { getAuthorizedClients, getSecretConfig } from "../utils.js"; import { emailMembershipPassHandler } from "./emailMembershipPassHandler.js"; import RedisModule from "ioredis"; import { setKey } from "api/functions/redisCache.js"; +import { createRedisModule } from "api/redis.js"; export const provisionNewMemberHandler: SQSHandlerFunction< AvailableSQSFunctions.ProvisionNewMember @@ -30,7 +31,11 @@ export const provisionNewMemberHandler: SQSHandlerFunction< logger, commonConfig, }); - const redisClient = new RedisModule.default(secretConfig.redis_url); + const redisClient = await createRedisModule( + secretConfig.redis_url, + secretConfig.fallback_redis_url, + logger, + ); const netId = email.replace("@illinois.edu", ""); const cacheKey = `membership:${netId}:acmpaid`; logger.info("Got authorized clients and Entra ID token."); diff --git a/src/common/config.ts b/src/common/config.ts index 6d10cb22..9ccee37d 100644 --- a/src/common/config.ts +++ b/src/common/config.ts @@ -189,6 +189,7 @@ export type SecretConfig = { stripe_endpoint_secret: string; stripe_links_endpoint_secret: string; redis_url: string; + fallback_redis_url: string; encryption_key: string; UIN_HASHING_SECRET_PEPPER: string; github_pat: string;