From 0faaaf0e2e0eb11d750579958e655ec94b2ee9f3 Mon Sep 17 00:00:00 2001 From: Konstantin Wohlwend Date: Wed, 19 Nov 2025 00:19:30 -0800 Subject: [PATCH 01/65] Email outbox backend --- .vscode/settings.json | 1 + apps/backend/package.json | 5 +- .../20251020180000_email_outbox/migration.sql | 243 +++++++ .../migration.sql | 112 ++++ apps/backend/prisma/schema.prisma | 173 ++++- apps/backend/scripts/run-email-queue.ts | 28 + .../otp/sign-in/verification-code-handler.tsx | 5 +- .../reset/verification-code-handler.tsx | 5 +- .../latest/auth/password/sign-up/route.tsx | 1 + .../send-verification-code/route.tsx | 1 + .../send-verification-code/route.tsx | 1 + .../verify/verification-code-handler.tsx | 7 +- .../src/app/api/latest/emails/README.md | 43 ++ .../api/latest/emails/delivery-info/route.tsx | 65 ++ .../api/latest/emails/render-email/route.tsx | 4 +- .../api/latest/emails/send-email/route.tsx | 232 ++----- .../credential-scanning/revoke/route.tsx | 13 +- .../internal/email-queue-step/route.tsx | 40 ++ .../app/api/latest/internal/emails/crud.tsx | 34 +- .../internal/failed-emails-digest/crud.tsx | 39 +- .../internal/failed-emails-digest/route.ts | 12 +- .../send-sign-in-invitation/route.tsx | 5 +- .../latest/internal/send-test-email/route.tsx | 4 +- .../accept/verification-code-handler.tsx | 5 +- apps/backend/src/lib/email-delivery-stats.tsx | 101 +++ apps/backend/src/lib/email-queue-step.tsx | 619 ++++++++++++++++++ apps/backend/src/lib/email-queue.tsx | 27 + apps/backend/src/lib/email-rendering.test.tsx | 466 +++++++++++++ apps/backend/src/lib/email-rendering.tsx | 127 +++- apps/backend/src/lib/emails-low-level.tsx | 350 ++++++++++ apps/backend/src/lib/emails.tsx | 454 ++----------- .../src/lib/notification-categories.ts | 17 +- apps/backend/src/lib/upstash.tsx | 7 +- apps/backend/src/prisma-client.tsx | 57 +- .../route-handlers/smart-route-handler.tsx | 16 +- apps/backend/src/utils/telemetry.tsx | 2 +- apps/dashboard/tsconfig.json | 2 +- apps/dev-launchpad/public/index.html | 8 + apps/e2e/tests/backend/backend-helpers.ts | 21 +- .../api/v1/auth/email-normalization.test.ts | 15 +- .../api/v1/emails/delivery-info.test.ts | 116 ++++ .../endpoints/api/v1/internal/email.test.ts | 24 - .../v1/internal/failed-emails-digest.test.ts | 225 ++++--- .../endpoints/api/v1/send-email.test.ts | 56 +- apps/e2e/tests/helpers.ts | 19 +- apps/e2e/tests/js/email.test.ts | 56 +- apps/e2e/tests/js/js-helpers.ts | 9 +- apps/e2e/tests/setup.ts | 11 +- docker/dependencies/docker.compose.yaml | 19 + package.json | 2 +- .../stack-shared/src/interface/crud/emails.ts | 2 - .../src/interface/server-interface.ts | 25 + packages/stack-shared/src/utils/results.tsx | 12 +- .../apps/implementations/server-app-impl.ts | 16 +- .../stack-app/apps/interfaces/server-app.ts | 7 +- .../template/src/lib/stack-app/email/index.ts | 23 + vercel.json | 6 + 57 files changed, 3117 insertions(+), 878 deletions(-) create mode 100644 apps/backend/prisma/migrations/20251020180000_email_outbox/migration.sql create mode 100644 apps/backend/prisma/migrations/20251020183000_migrate_sent_email/migration.sql create mode 100644 apps/backend/scripts/run-email-queue.ts create mode 100644 apps/backend/src/app/api/latest/emails/README.md create mode 100644 apps/backend/src/app/api/latest/emails/delivery-info/route.tsx create mode 100644 apps/backend/src/app/api/latest/internal/email-queue-step/route.tsx create mode 100644 apps/backend/src/lib/email-delivery-stats.tsx create mode 100644 apps/backend/src/lib/email-queue-step.tsx create mode 100644 apps/backend/src/lib/email-queue.tsx create mode 100644 apps/backend/src/lib/email-rendering.test.tsx create mode 100644 apps/backend/src/lib/emails-low-level.tsx create mode 100644 apps/e2e/tests/backend/endpoints/api/v1/emails/delivery-info.test.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index 5baf0f4354..f5bf18b958 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -16,6 +16,7 @@ "Ciphertext", "cjsx", "clsx", + "dbgenerated", "cmdk", "codegen", "crockford", diff --git a/apps/backend/package.json b/apps/backend/package.json index 74815dc5a3..7b4298ed98 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -7,7 +7,7 @@ "typecheck": "tsc --noEmit", "with-env": "dotenv -c development --", "with-env:prod": "dotenv -c --", - "dev": "concurrently -n \"dev,codegen,prisma-studio\" -k \"next dev --turbopack --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\"", + "dev": "concurrently -n \"dev,codegen,prisma-studio,email-queue\" -k \"next dev --turbopack --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02\" \"pnpm run codegen:watch\" \"pnpm run prisma-studio\" \"pnpm run run-email-queue\"", "build": "pnpm run codegen && next build", "docker-build": "pnpm run codegen && next build --experimental-build-mode compile", "build-self-host-migration-script": "tsup --config scripts/db-migrations.tsup.config.ts", @@ -35,7 +35,8 @@ "generate-openapi-fumadocs": "pnpm run with-env tsx scripts/generate-openapi-fumadocs.ts", "generate-keys": "pnpm run with-env tsx scripts/generate-keys.ts", "db-seed-script": "pnpm run db:seed", - "verify-data-integrity": "pnpm run with-env tsx scripts/verify-data-integrity.ts" + "verify-data-integrity": "pnpm run with-env tsx scripts/verify-data-integrity.ts", + "run-email-queue": "pnpm run with-env tsx scripts/run-email-queue.ts" }, "prisma": { "seed": "pnpm run db-seed-script" diff --git a/apps/backend/prisma/migrations/20251020180000_email_outbox/migration.sql b/apps/backend/prisma/migrations/20251020180000_email_outbox/migration.sql new file mode 100644 index 0000000000..081df13109 --- /dev/null +++ b/apps/backend/prisma/migrations/20251020180000_email_outbox/migration.sql @@ -0,0 +1,243 @@ +-- CreateEnum +CREATE TYPE "EmailOutboxStatus" AS ENUM ( + 'PAUSED', + 'PREPARING', + 'RENDERING', + 'RENDER_ERROR', + 'SCHEDULED', + 'QUEUED', + 'SENDING', + 'SERVER_ERROR', + 'SENT', + 'DELIVERY_DELAYED', + 'BOUNCED', + 'OPENED', + 'CLICKED', + 'MARKED_AS_SPAM' +); + +-- CreateEnum +CREATE TYPE "EmailOutboxSimpleStatus" AS ENUM ('IN_PROGRESS', 'ERROR', 'OK'); + +-- CreateEnum +CREATE TYPE "EmailOutboxSkippedReason" AS ENUM ('USER_UNSUBSCRIBED', 'USER_DELETED_ACCOUNT'); + +-- CreateEnum +CREATE TYPE "EmailOutboxCreatedWith" AS ENUM ('DRAFT', 'PROGRAMMATIC_CALL'); + +-- CreateTable +CREATE TABLE "EmailOutbox" ( + "tenancyId" UUID NOT NULL, + "id" UUID NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "tsxSource" TEXT NOT NULL, + "themeId" TEXT, + "isHighPriority" BOOLEAN NOT NULL, + "to" JSONB NOT NULL, + "renderedIsTransactional" BOOLEAN, + "renderedNotificationCategoryId" TEXT, + "extraRenderVariables" JSONB NOT NULL, + "createdWith" "EmailOutboxCreatedWith" NOT NULL, + "emailDraftId" TEXT, + "emailProgrammaticCallTemplateId" TEXT, + "shouldSkipDeliverabilityCheck" BOOLEAN NOT NULL, + "status" "EmailOutboxStatus" NOT NULL GENERATED ALWAYS AS ( + CASE + WHEN "isPaused" THEN 'PAUSED'::"EmailOutboxStatus" + WHEN "markedAsSpamAt" IS NOT NULL THEN 'MARKED_AS_SPAM'::"EmailOutboxStatus" + WHEN "clickedAt" IS NOT NULL THEN 'CLICKED'::"EmailOutboxStatus" + WHEN "openedAt" IS NOT NULL THEN 'OPENED'::"EmailOutboxStatus" + WHEN "canHaveDeliveryInfo" IS TRUE AND "bouncedAt" IS NOT NULL THEN 'BOUNCED'::"EmailOutboxStatus" + WHEN "canHaveDeliveryInfo" IS TRUE AND "deliveryDelayedAt" IS NOT NULL THEN 'DELIVERY_DELAYED'::"EmailOutboxStatus" + WHEN "finishedSendingAt" IS NOT NULL AND ( + "sendServerErrorExternalMessage" IS NOT NULL + OR "sendServerErrorExternalDetails" IS NOT NULL + OR "sendServerErrorInternalMessage" IS NOT NULL + OR "sendServerErrorInternalDetails" IS NOT NULL + ) THEN 'SERVER_ERROR'::"EmailOutboxStatus" + WHEN "finishedSendingAt" IS NOT NULL THEN 'SENT'::"EmailOutboxStatus" + WHEN "startedSendingAt" IS NOT NULL THEN 'SENDING'::"EmailOutboxStatus" + WHEN "finishedRenderingAt" IS NOT NULL AND ( + "renderErrorExternalMessage" IS NOT NULL + OR "renderErrorExternalDetails" IS NOT NULL + OR "renderErrorInternalMessage" IS NOT NULL + OR "renderErrorInternalDetails" IS NOT NULL + ) THEN 'RENDER_ERROR'::"EmailOutboxStatus" + WHEN "finishedRenderingAt" IS NOT NULL AND "renderedHtml" IS NOT NULL AND "isQueued" THEN 'QUEUED'::"EmailOutboxStatus" + WHEN "finishedRenderingAt" IS NOT NULL AND "renderedHtml" IS NOT NULL THEN 'SCHEDULED'::"EmailOutboxStatus" + WHEN "startedRenderingAt" IS NOT NULL THEN 'RENDERING'::"EmailOutboxStatus" + ELSE 'PREPARING'::"EmailOutboxStatus" + END + ) STORED, + "simpleStatus" "EmailOutboxSimpleStatus" NOT NULL GENERATED ALWAYS AS ( + CASE + WHEN "renderErrorExternalMessage" IS NOT NULL OR "sendServerErrorExternalMessage" IS NOT NULL OR "bouncedAt" IS NOT NULL THEN 'ERROR'::"EmailOutboxSimpleStatus" + WHEN "finishedSendingAt" IS NOT NULL AND ("skippedReason" IS NOT NULL OR "canHaveDeliveryInfo" IS FALSE OR "deliveredAt" IS NOT NULL) THEN 'OK'::"EmailOutboxSimpleStatus" + WHEN "finishedSendingAt" IS NULL OR ("canHaveDeliveryInfo" IS TRUE AND "deliveredAt" IS NULL) THEN 'IN_PROGRESS'::"EmailOutboxSimpleStatus" + ELSE 'OK'::"EmailOutboxSimpleStatus" + END + ) STORED, + "priority" INTEGER NOT NULL GENERATED ALWAYS AS ( + (CASE WHEN "isHighPriority" THEN 100 ELSE 0 END) + + (CASE WHEN "renderedIsTransactional" THEN 10 ELSE 0 END) + ) STORED, + "isPaused" BOOLEAN NOT NULL DEFAULT FALSE, + "renderedByWorkerId" UUID, + "startedRenderingAt" TIMESTAMP(3), + "finishedRenderingAt" TIMESTAMP(3), + "renderErrorExternalMessage" TEXT, + "renderErrorExternalDetails" JSONB, + "renderErrorInternalMessage" TEXT, + "renderErrorInternalDetails" JSONB, + "renderedHtml" TEXT, + "renderedText" TEXT, + "renderedSubject" TEXT, + "scheduledAt" TIMESTAMP(3) NOT NULL, + "isQueued" BOOLEAN NOT NULL DEFAULT FALSE, + "scheduledAtIfNotYetQueued" TIMESTAMP(3) GENERATED ALWAYS AS ( + CASE WHEN "isQueued" THEN NULL ELSE "scheduledAt" END + ) STORED, + "startedSendingAt" TIMESTAMP(3), + "finishedSendingAt" TIMESTAMP(3), + "sentAt" TIMESTAMP(3) GENERATED ALWAYS AS ( + CASE + WHEN "canHaveDeliveryInfo" IS TRUE THEN "deliveredAt" + WHEN "canHaveDeliveryInfo" IS FALSE THEN "finishedSendingAt" + ELSE NULL + END + ) STORED, + "sendServerErrorExternalMessage" TEXT, + "sendServerErrorExternalDetails" JSONB, + "sendServerErrorInternalMessage" TEXT, + "sendServerErrorInternalDetails" JSONB, + "skippedReason" "EmailOutboxSkippedReason", + "canHaveDeliveryInfo" BOOLEAN, + "deliveredAt" TIMESTAMP(3), + "deliveryDelayedAt" TIMESTAMP(3), + "bouncedAt" TIMESTAMP(3), + "openedAt" TIMESTAMP(3), + "clickedAt" TIMESTAMP(3), + "unsubscribedAt" TIMESTAMP(3), + "markedAsSpamAt" TIMESTAMP(3), + + CONSTRAINT "EmailOutbox_pkey" PRIMARY KEY ("tenancyId", "id"), + CONSTRAINT "EmailOutbox_tenancyId_fkey" FOREIGN KEY ("tenancyId") REFERENCES "Tenancy"("id") ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT "EmailOutbox_render_worker_pair_check" + CHECK (("renderedByWorkerId" IS NULL) = ("startedRenderingAt" IS NULL)), + CONSTRAINT "EmailOutbox_finished_rendering_check" + CHECK ("finishedRenderingAt" IS NULL OR "startedRenderingAt" IS NOT NULL), + CONSTRAINT "EmailOutbox_render_payload_when_not_finished_check" + CHECK ( + "finishedRenderingAt" IS NOT NULL OR ( + "renderedHtml" IS NULL + AND "renderedText" IS NULL + AND "renderedSubject" IS NULL + AND "renderedIsTransactional" IS NULL + AND "renderedNotificationCategoryId" IS NULL + AND "renderErrorExternalMessage" IS NULL + AND "renderErrorExternalDetails" IS NULL + AND "renderErrorInternalMessage" IS NULL + AND "renderErrorInternalDetails" IS NULL + ) + ), + CONSTRAINT "EmailOutbox_render_payload_consistency_check" + CHECK ( + "finishedRenderingAt" IS NULL OR ( + ( + ("renderedHtml" IS NOT NULL OR "renderedText" IS NOT NULL OR "renderedSubject" IS NOT NULL OR "renderedIsTransactional" IS NOT NULL OR "renderedNotificationCategoryId" IS NOT NULL) + AND "renderErrorExternalMessage" IS NULL + AND "renderErrorExternalDetails" IS NULL + AND "renderErrorInternalMessage" IS NULL + AND "renderErrorInternalDetails" IS NULL + ) OR ( + ("renderedHtml" IS NULL AND "renderedText" IS NULL AND "renderedSubject" IS NULL AND "renderedIsTransactional" IS NULL AND "renderedNotificationCategoryId" IS NULL) + AND ( + "renderErrorExternalMessage" IS NOT NULL + OR "renderErrorExternalDetails" IS NOT NULL + OR "renderErrorInternalMessage" IS NOT NULL + OR "renderErrorInternalDetails" IS NOT NULL + ) + ) + ) + ), + CONSTRAINT "EmailOutbox_email_draft_check" + CHECK ("createdWith" <> 'DRAFT' OR "emailDraftId" IS NOT NULL), + CONSTRAINT "EmailOutbox_finished_sending_check" + CHECK ("finishedSendingAt" IS NULL OR "startedSendingAt" IS NOT NULL), + CONSTRAINT "EmailOutbox_send_payload_when_not_finished_check" + CHECK ( + "finishedSendingAt" IS NOT NULL OR ( + "sendServerErrorExternalMessage" IS NULL + AND "sendServerErrorExternalDetails" IS NULL + AND "sendServerErrorInternalMessage" IS NULL + AND "sendServerErrorInternalDetails" IS NULL + AND "skippedReason" IS NULL + AND "canHaveDeliveryInfo" IS NULL + AND "deliveredAt" IS NULL + AND "deliveryDelayedAt" IS NULL + AND "bouncedAt" IS NULL + AND "openedAt" IS NULL + AND "clickedAt" IS NULL + AND "unsubscribedAt" IS NULL + AND "markedAsSpamAt" IS NULL + ) + ), + CONSTRAINT "EmailOutbox_can_have_delivery_info_check" + CHECK ( + ("finishedSendingAt" IS NULL AND "canHaveDeliveryInfo" IS NULL) + OR ("finishedSendingAt" IS NOT NULL AND "canHaveDeliveryInfo" IS NOT NULL) + ), + CONSTRAINT "EmailOutbox_delivery_status_check" + CHECK ( + "canHaveDeliveryInfo" IS DISTINCT FROM FALSE OR ( + "deliveredAt" IS NULL + AND "deliveryDelayedAt" IS NULL + AND "bouncedAt" IS NULL + ) + ), + CONSTRAINT "EmailOutbox_delivery_exclusive_check" + CHECK ( + (CASE WHEN "deliveredAt" IS NOT NULL THEN 1 ELSE 0 END) + + (CASE WHEN "deliveryDelayedAt" IS NOT NULL THEN 1 ELSE 0 END) + + (CASE WHEN "bouncedAt" IS NOT NULL THEN 1 ELSE 0 END) + <= 1 + ), + CONSTRAINT "EmailOutbox_click_implies_open_check" + CHECK ("clickedAt" IS NULL OR "openedAt" IS NOT NULL) +); + +-- CreateTable +CREATE TABLE "EmailOutboxProcessingMetadata" ( + "key" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "lastExecutedAt" TIMESTAMP(3), + + CONSTRAINT "EmailOutboxProcessingMetadata_pkey" PRIMARY KEY ("key") +); + +-- CreateIndex +CREATE INDEX "EmailOutbox_status_tenancy_idx" ON "EmailOutbox" ("tenancyId", "status"); + +-- CreateIndex +CREATE INDEX "EmailOutbox_simple_status_tenancy_idx" ON "EmailOutbox" ("tenancyId", "simpleStatus"); + +-- CreateIndex +CREATE INDEX "EmailOutbox_render_queue_idx" ON "EmailOutbox" ("tenancyId", "createdAt") WHERE "renderedByWorkerId" IS NULL; + +-- CreateIndex +CREATE INDEX "EmailOutbox_schedule_idx" ON "EmailOutbox" ("tenancyId", "scheduledAt") WHERE NOT "isQueued"; + +-- CreateIndex +CREATE INDEX "EmailOutbox_sending_idx" ON "EmailOutbox" ("tenancyId", "priority", "scheduledAt") WHERE "isQueued" AND "startedSendingAt" IS NULL; + +-- CreateIndex +CREATE INDEX "EmailOutbox_ordering_idx" + ON "EmailOutbox" ( + "tenancyId", + "finishedSendingAt" DESC NULLS FIRST, + "scheduledAtIfNotYetQueued" DESC NULLS LAST, + "priority" ASC, + "id" ASC + ); diff --git a/apps/backend/prisma/migrations/20251020183000_migrate_sent_email/migration.sql b/apps/backend/prisma/migrations/20251020183000_migrate_sent_email/migration.sql new file mode 100644 index 0000000000..63df0ef504 --- /dev/null +++ b/apps/backend/prisma/migrations/20251020183000_migrate_sent_email/migration.sql @@ -0,0 +1,112 @@ +INSERT INTO "EmailOutbox" ( + "tenancyId", + "id", + "createdAt", + "updatedAt", + "tsxSource", + "themeId", + "renderedIsTransactional", + "isHighPriority", + "to", + "renderedNotificationCategoryId", + "extraRenderVariables", + "createdWith", + "emailDraftId", + "emailProgrammaticCallTemplateId", + "isPaused", + "renderedByWorkerId", + "startedRenderingAt", + "finishedRenderingAt", + "renderErrorExternalMessage", + "renderErrorExternalDetails", + "renderErrorInternalMessage", + "renderErrorInternalDetails", + "renderedHtml", + "renderedText", + "renderedSubject", + "scheduledAt", + "isQueued", + "startedSendingAt", + "finishedSendingAt", + "sendServerErrorExternalMessage", + "sendServerErrorExternalDetails", + "sendServerErrorInternalMessage", + "sendServerErrorInternalDetails", + "skippedReason", + "canHaveDeliveryInfo", + "deliveredAt", + "deliveryDelayedAt", + "bouncedAt", + "openedAt", + "clickedAt", + "unsubscribedAt", + "markedAsSpamAt", + "shouldSkipDeliverabilityCheck" +) +SELECT + se."tenancyId", + se."id", + se."createdAt", + se."updatedAt", + 'export function LegacyEmail() { throw new Error("This is a legacy email older than the EmailOutbox migration. Its tsx source code is no longer available."); }' AS "tsxSource", + NULL, + TRUE, + FALSE, + jsonb_build_object( + 'type', 'custom-emails', + 'emails', COALESCE(to_jsonb(se."to"), '[]'::jsonb) + ), + NULL, + NULL, + 'PROGRAMMATIC_CALL', + NULL, + NULL, + FALSE, + gen_random_uuid(), + se."createdAt", + se."createdAt", + NULL, + NULL, + NULL, + NULL, + se."html", + se."text", + se."subject", + se."createdAt", + TRUE, + se."createdAt", + se."updatedAt", + CASE + WHEN se."error" IS NULL THEN NULL + ELSE COALESCE(se."error"->>'message', 'An unknown error occurred while sending the email.') + END, + CASE + WHEN se."error" IS NULL THEN NULL + ELSE jsonb_strip_nulls(jsonb_build_object( + 'legacyErrorType', se."error"->>'errorType', + 'legacyCanRetry', se."error"->>'canRetry' + )) + END, + CASE + WHEN se."error" IS NULL THEN NULL + ELSE COALESCE(se."error"->>'message', se."error"->>'errorType', 'Legacy send error') + END, + se."error", + NULL, + FALSE, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + FALSE +FROM "SentEmail" se +ON CONFLICT ("tenancyId", "id") DO NOTHING; + +INSERT INTO "EmailOutboxProcessingMetadata" ("key", "createdAt", "updatedAt", "lastExecutedAt") +VALUES ('email-queue-step', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL) +ON CONFLICT ("key") DO NOTHING; + +DROP TABLE IF EXISTS "SentEmail"; diff --git a/apps/backend/prisma/schema.prisma b/apps/backend/prisma/schema.prisma index fb9228d9ca..e0246f9f7a 100644 --- a/apps/backend/prisma/schema.prisma +++ b/apps/backend/prisma/schema.prisma @@ -45,8 +45,9 @@ model Tenancy { branchId String // If organizationId is NULL, hasNoOrganization must be TRUE. If organizationId is not NULL, hasNoOrganization must be NULL. - organizationId String? @db.Uuid + organizationId String? @db.Uuid hasNoOrganization BooleanTrue? + emailOutboxes EmailOutbox[] @@unique([projectId, branchId, organizationId]) @@unique([projectId, branchId, hasNoOrganization]) @@ -179,7 +180,6 @@ model ProjectUser { passkeyAuthMethod PasskeyAuthMethod[] otpAuthMethod OtpAuthMethod[] oauthAuthMethod OAuthAuthMethod[] - SentEmail SentEmail[] projectApiKey ProjectApiKey[] directPermissions ProjectUserDirectPermission[] Project Project? @relation(fields: [projectId], references: [id]) @@ -651,28 +651,175 @@ model EventIpInfo { //#endregion -model SentEmail { +enum EmailOutboxStatus { + PAUSED + PREPARING + RENDERING + RENDER_ERROR + SCHEDULED + QUEUED + SENDING + SERVER_ERROR + SENT + DELIVERY_DELAYED + BOUNCED + OPENED + CLICKED + MARKED_AS_SPAM +} + +enum EmailOutboxSimpleStatus { + IN_PROGRESS + ERROR + OK +} + +enum EmailOutboxSkippedReason { + USER_UNSUBSCRIBED + USER_DELETED_ACCOUNT +} + +enum EmailOutboxCreatedWith { + DRAFT + PROGRAMMATIC_CALL +} + +// In most displays, the way emails in the outbox should be ordered is: +// - by finishedSendingAt, descending (null comes first) +// - by scheduledAtIfNotYetQueued, descending (null comes last) +// - by priority, ascending +// - by id, ascending +model EmailOutbox { tenancyId String @db.Uuid id String @default(uuid()) @db.Uuid - userId String? @db.Uuid - createdAt DateTime @default(now()) updatedAt DateTime @updatedAt - senderConfig Json - to String[] - subject String - html String? - text String? - - error Json? - user ProjectUser? @relation(fields: [tenancyId, userId], references: [tenancyId, projectUserId], onDelete: Cascade) + // note: [tsxSource, themeId, isHighPriority, toUserIds, toEmails, extraRenderVariables, shouldSkipDeliverabilityCheck] can be changed, but only while startedSendingAt is not set, and any modification should reset fields like [renderedByWorkerId, startedRenderingAt, finishedRenderingAt, renderError*, rendered*, isQueued] and any other fields with similar semantics to null. + tsxSource String + themeId String? + isHighPriority Boolean + // Who the email is being sent to. { type: "user-primary-email", userId: string } | { type: "user-custom-emails", userId: string, emails: string[] } | { type: "custom-emails", emails: string[] } + to Json + extraRenderVariables Json + shouldSkipDeliverabilityCheck Boolean + + createdWith EmailOutboxCreatedWith + // If the email was created from a draft, it is kept around so we can later group by it. Must be set if and only if createdWith is DRAFT. + emailDraftId String? + // If the email was created from a template programmatically, it is kept around so we can later group by it. Must be NOT set if createdWith is NOT PROGRAMMATIC_CALL. If createdWith is PROGRAMMATIC_CALL and this is not set, then no template was used (eg. email was sent directly as HTML). + emailProgrammaticCallTemplateId String? + + // Computed from EmailOutbox can be the `status` of the email: + // + // - ⚪ `paused` : isPaused + // - ⚪ `preparing` : !isPaused && !startedRenderingAt + // - ⚪ `rendering` : !isPaused && !finishedRenderingAt + // - 🔴 `render-error` : !isPaused && finishedRenderingAt && renderError + // - ⚪ `scheduled` : !isPaused && finishedRenderingAt && !renderError && !isQueued + // - ⚪ `queued` : !isPaused && finishedRenderingAt && !renderError && isQueued && !startedSendingAt + // - ⚪ `sending` : !isPaused && startedSendingAt && !deliveredAt + // - 🔴 `server-error` : !isPaused && finishedSendingAt && sendServerErrorMessage + // - ⚫ `skipped` : !isPaused && finishedSendingAt && !sendServerErrorMessage && skippedReason + // - 🟢 `sent` : !isPaused && finishedSendingAt && !openedAt && !markedAsSpamAt && !sendServerErrorMessage && !skippedReason && (canHaveDeliveryInfo ? deliveredAt : finishedSendingAt) + // - ⚪ `delivery-delayed` : !isPaused && canHaveDeliveryInfo && deliveryDelayedAt + // - 🔴 `bounced` : !isPaused && canHaveDeliveryInfo && bouncedAt + // - 🔵 `opened` : !isPaused && openedAt && !clickedAt && !markedAsSpamAt + // - 🟣 `clicked` : !isPaused && clickedAt && !markedAsSpamAt + // - 🟡 `marked-as-spam` : !isPaused && markedAsSpamAt + // + // This column is auto-generated as defined in the SQL migration. It can not be set manually. + status EmailOutboxStatus @default(dbgenerated("")) + + // A simplified version of the status property. + // In terms of the color mapping of `status`, white statuses have a `simpleStatus` of `in-progress`, red statuses have a `simpleStatus` of `error`, and everything else has a `simpleStatus` of `ok`. + // + // This column is auto-generated as defined in the SQL migration. It can not be set manually. + simpleStatus EmailOutboxSimpleStatus @default(dbgenerated("")) + + // priority is the sending priority of the email among all emails that are not yet sent but already past their scheduled time. Higher priority means it should be sent sooner. + // + // This column is auto-generated based on the following formula: + // priority = (isHighPriority ? 100 : 0) + (renderedIsTransactional ? 10 : 0) + priority Int @default(dbgenerated("")) + + // This column is auto-generated as defined in the SQL migration. It can not be set manually. + isPaused Boolean @default(false) + + // either both or neither of [renderedByWorkerId, startedRenderingAt] must be set + renderedByWorkerId String? @db.Uuid + startedRenderingAt DateTime? + // if startedRenderingAt is not set, then finishedRenderingAt is also not set + finishedRenderingAt DateTime? + + // if finishedRenderingAt is set, then exactly one of [renderedHtml, renderedText, renderedSubject, renderedIsTransactional, renderedNotificationCategoryId] or [renderErrorExternalMessage, renderErrorExternalDetails, renderErrorInternalMessage, renderErrorInternalDetails] must be set; if finishedRenderingAt is not set, then none of the aforementioned fields are set + renderErrorExternalMessage String? + renderErrorExternalDetails Json? + renderErrorInternalMessage String? + renderErrorInternalDetails Json? + renderedHtml String? + renderedText String? + renderedSubject String? + renderedIsTransactional Boolean? + renderedNotificationCategoryId String? + + // The scheduled time of when the email should be added to the queue. Can be edited, but only if the email has not yet started sending. Doing so should set isQueued to false. + scheduledAt DateTime + + // The scheduled time of the email if it is in the future. + isQueued Boolean @default(false) + + // A generated column that is equal to scheduledAt if isQueued is false, otherwise null. + scheduledAtIfNotYetQueued DateTime? @default(dbgenerated("")) + + // if finishedRenderingAt is not set, then startedSendingAt is also not set + startedSendingAt DateTime? + // if startedSendingAt is not set, then finishedSendingAt is also not set + finishedSendingAt DateTime? + + // A generated column that is equal to finishedSendingAt if canHaveDeliveryInfo is true, otherwise deliveredAt. + sentAt DateTime? @default(dbgenerated("")) + + // either all or none of [sendServerErrorExternalMessage, sendServerErrorExternalDetails, sendServerErrorInternalMessage, sendServerErrorInternalDetails] must be set. If finishedSendingAt is not set, then none of these are set. + sendServerErrorExternalMessage String? + sendServerErrorExternalDetails Json? + sendServerErrorInternalMessage String? + sendServerErrorInternalDetails Json? + + // The reason why the email was skipped. If finishedSendingAt is not set, then this is also not set. Usually one of: + skippedReason EmailOutboxSkippedReason? + + // Whether this email was sent through a server that provides delivery info. This is set if and only if finishedSendingAt is set (it is only determined once the email has been sent). If canHaveDeliveryInfo is false, then [deliveredAt, deliveryDelayedAt, bouncedAt] are all not set. This flag is usually set to true if the email provider is Resend. + canHaveDeliveryInfo Boolean? + + // at most one of [deliveredAt, deliveryDelayedAt, bouncedAt] can be set. If finishedSendingAt is not set, then none of these are set. + deliveredAt DateTime? + deliveryDelayedAt DateTime? + bouncedAt DateTime? + + // if finishedSendingAt is not set, then openedAt is also not set + openedAt DateTime? + // note: setting clickedAt should also set openedAt if it's not set yet + clickedAt DateTime? + unsubscribedAt DateTime? + markedAsSpamAt DateTime? + + tenancy Tenancy @relation(fields: [tenancyId], references: [id], onDelete: Cascade) @@id([tenancyId, id]) } +model EmailOutboxProcessingMetadata { + key String @id + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + lastExecutedAt DateTime? +} + model EmailDraft { tenancyId String @db.Uuid diff --git a/apps/backend/scripts/run-email-queue.ts b/apps/backend/scripts/run-email-queue.ts new file mode 100644 index 0000000000..fbc8764b5d --- /dev/null +++ b/apps/backend/scripts/run-email-queue.ts @@ -0,0 +1,28 @@ +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; +import { runAsynchronously } from "@stackframe/stack-shared/dist/utils/promises"; + +async function main() { + console.log("Starting email queue processor..."); + const cronSecret = getEnvVariable('CRON_SECRET'); + if (!cronSecret) throw new Error("CRON_SECRET environment variable is not set."); + + const baseUrl = `http://localhost:${getEnvVariable('NEXT_PUBLIC_STACK_PORT_PREFIX', '81')}02`; + + setInterval(() => { + runAsynchronously(async () => { + console.log("Running email queue step..."); + const res = await fetch(`${baseUrl}/api/latest/internal/email-queue-step`, { + headers: { 'Authorization': `Bearer ${cronSecret}` }, + }); + if (!res.ok) throw new StackAssertionError(`Failed to call email queue step: ${res.status} ${res.statusText}\n${await res.text()}`, { res }); + console.log("Email queue step completed."); + }); + }, 60000); +} + +// eslint-disable-next-line no-restricted-syntax +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/apps/backend/src/app/api/latest/auth/otp/sign-in/verification-code-handler.tsx b/apps/backend/src/app/api/latest/auth/otp/sign-in/verification-code-handler.tsx index 69e54111e0..3e1a913a63 100644 --- a/apps/backend/src/app/api/latest/auth/otp/sign-in/verification-code-handler.tsx +++ b/apps/backend/src/app/api/latest/auth/otp/sign-in/verification-code-handler.tsx @@ -1,5 +1,5 @@ import { getAuthContactChannelWithEmailNormalization } from "@/lib/contact-channel"; -import { sendEmailFromTemplate } from "@/lib/emails"; +import { sendEmailFromDefaultTemplate } from "@/lib/emails"; import { getSoleTenancyFromProjectBranch, Tenancy } from "@/lib/tenancies"; import { createAuthTokens } from "@/lib/tokens"; import { createOrUpgradeAnonymousUser } from "@/lib/users"; @@ -85,7 +85,7 @@ export const signInVerificationCodeHandler = createVerificationCodeHandler({ }), async send(codeObj, createOptions, sendOptions: { email: string }) { const tenancy = await getSoleTenancyFromProjectBranch(createOptions.project.id, createOptions.branchId); - await sendEmailFromTemplate({ + await sendEmailFromDefaultTemplate({ tenancy, email: createOptions.method.email, user: null, @@ -94,6 +94,7 @@ export const signInVerificationCodeHandler = createVerificationCodeHandler({ magicLink: codeObj.link.toString(), otp: codeObj.code.slice(0, 6).toUpperCase(), }, + shouldSkipDeliverabilityCheck: true, version: createOptions.method.type === "legacy" ? 1 : undefined, }); diff --git a/apps/backend/src/app/api/latest/auth/password/reset/verification-code-handler.tsx b/apps/backend/src/app/api/latest/auth/password/reset/verification-code-handler.tsx index 532b42ade9..fe0e51c389 100644 --- a/apps/backend/src/app/api/latest/auth/password/reset/verification-code-handler.tsx +++ b/apps/backend/src/app/api/latest/auth/password/reset/verification-code-handler.tsx @@ -1,4 +1,4 @@ -import { sendEmailFromTemplate } from "@/lib/emails"; +import { sendEmailFromDefaultTemplate } from "@/lib/emails"; import { getSoleTenancyFromProjectBranch } from "@/lib/tenancies"; import { createVerificationCodeHandler } from "@/route-handlers/verification-code-handler"; import { VerificationCodeType } from "@prisma/client"; @@ -37,7 +37,7 @@ export const resetPasswordVerificationCodeHandler = createVerificationCodeHandle }), async send(codeObj, createOptions, sendOptions: { user: UsersCrud["Admin"]["Read"] }) { const tenancy = await getSoleTenancyFromProjectBranch(createOptions.project.id, createOptions.branchId); - await sendEmailFromTemplate({ + await sendEmailFromDefaultTemplate({ tenancy, user: sendOptions.user, email: createOptions.method.email, @@ -45,6 +45,7 @@ export const resetPasswordVerificationCodeHandler = createVerificationCodeHandle extraVariables: { passwordResetLink: codeObj.link.toString(), }, + shouldSkipDeliverabilityCheck: true, }); }, async handler(tenancy, { email }, data, { password }) { diff --git a/apps/backend/src/app/api/latest/auth/password/sign-up/route.tsx b/apps/backend/src/app/api/latest/auth/password/sign-up/route.tsx index bc9ad092a3..f8689d2c6a 100644 --- a/apps/backend/src/app/api/latest/auth/password/sign-up/route.tsx +++ b/apps/backend/src/app/api/latest/auth/password/sign-up/route.tsx @@ -79,6 +79,7 @@ export const POST = createSmartRouteHandler({ callbackUrl: verificationCallbackUrl, }, { user: createdUser, + shouldSkipDeliverabilityCheck: false, }); })()); } diff --git a/apps/backend/src/app/api/latest/contact-channels/[user_id]/[contact_channel_id]/send-verification-code/route.tsx b/apps/backend/src/app/api/latest/contact-channels/[user_id]/[contact_channel_id]/send-verification-code/route.tsx index 4fe4590dc1..47d341e921 100644 --- a/apps/backend/src/app/api/latest/contact-channels/[user_id]/[contact_channel_id]/send-verification-code/route.tsx +++ b/apps/backend/src/app/api/latest/contact-channels/[user_id]/[contact_channel_id]/send-verification-code/route.tsx @@ -87,6 +87,7 @@ export const POST = createSmartRouteHandler({ callbackUrl, }, { user, + shouldSkipDeliverabilityCheck: true, }); return { diff --git a/apps/backend/src/app/api/latest/contact-channels/send-verification-code/route.tsx b/apps/backend/src/app/api/latest/contact-channels/send-verification-code/route.tsx index 3089c169f5..f8915851b1 100644 --- a/apps/backend/src/app/api/latest/contact-channels/send-verification-code/route.tsx +++ b/apps/backend/src/app/api/latest/contact-channels/send-verification-code/route.tsx @@ -42,6 +42,7 @@ export const POST = createSmartRouteHandler({ callbackUrl, }, { user, + shouldSkipDeliverabilityCheck: true, }); return { diff --git a/apps/backend/src/app/api/latest/contact-channels/verify/verification-code-handler.tsx b/apps/backend/src/app/api/latest/contact-channels/verify/verification-code-handler.tsx index e1879cca92..a9e0ff2538 100644 --- a/apps/backend/src/app/api/latest/contact-channels/verify/verification-code-handler.tsx +++ b/apps/backend/src/app/api/latest/contact-channels/verify/verification-code-handler.tsx @@ -1,4 +1,4 @@ -import { sendEmailFromTemplate } from "@/lib/emails"; +import { sendEmailFromDefaultTemplate } from "@/lib/emails"; import { getSoleTenancyFromProjectBranch } from "@/lib/tenancies"; import { getPrismaClientForTenancy } from "@/prisma-client"; import { createVerificationCodeHandler } from "@/route-handlers/verification-code-handler"; @@ -31,10 +31,10 @@ export const contactChannelVerificationCodeHandler = createVerificationCodeHandl statusCode: yupNumber().oneOf([200]).defined(), bodyType: yupString().oneOf(["success"]).defined(), }), - async send(codeObj, createOptions, sendOptions: { user: UsersCrud["Admin"]["Read"] }) { + async send(codeObj, createOptions, sendOptions: { user: UsersCrud["Admin"]["Read"], shouldSkipDeliverabilityCheck: boolean }) { const tenancy = await getSoleTenancyFromProjectBranch(createOptions.project.id, createOptions.branchId); - await sendEmailFromTemplate({ + await sendEmailFromDefaultTemplate({ tenancy, user: sendOptions.user, email: createOptions.method.email, @@ -42,6 +42,7 @@ export const contactChannelVerificationCodeHandler = createVerificationCodeHandl extraVariables: { emailVerificationLink: codeObj.link.toString(), }, + shouldSkipDeliverabilityCheck: sendOptions.shouldSkipDeliverabilityCheck, }); }, async handler(tenancy, { email }, data) { diff --git a/apps/backend/src/app/api/latest/emails/README.md b/apps/backend/src/app/api/latest/emails/README.md new file mode 100644 index 0000000000..d628933f64 --- /dev/null +++ b/apps/backend/src/app/api/latest/emails/README.md @@ -0,0 +1,43 @@ +# Email Infrastructure Overview + +This folder contains the HTTP endpoints that sit on top of the new email outbox +pipeline. The pipeline is intentionally asynchronous: instead of sending mail +inside request handlers we persist work items to the `EmailOutbox` table and let +background workers render, queue, and deliver them. + +## Execution Flow + +1. **Enqueue** – API endpoints (and server-side helpers) call + `sendEmailToAll` to persist one row per recipient. Each entry + captures the template source, render variables, target recipient, priority, + and scheduling metadata. We enqueue the QStash worker so the + pipeline continues in the background without blocking the caller. +2. **Render** – `runEmailQueueStep` acquires an advisory lock and atomically + claims rows that have not been rendered. Emails are rendered in tenancy + batches via Freestyle, producing HTML/Text/Subject snapshots while capturing + render errors in structured fields. +3. **Queue** – Rendered rows whose `scheduled_at` is in the past are marked as + ready (`isQueued = true`). Capacity is calculated per tenancy based on recent + delivery performance to decide how many emails can be handed off to the + sender during this iteration. +4. **Send** – Claimed rows are processed in parallel. Before delivery we fetch + the latest user data, honour notification preferences and skip users who have + unsubscribed or deleted their account. Provider responses are captured in the + `sendServerError*` fields so the dashboard can surface actionable feedback. +5. **Delivery Stats** – The worker updates `EmailOutboxProcessingMetadata` so we + can derive execution deltas and expose aggregated metrics via the + `/emails/delivery-info` endpoint. + +## Key Tables + +- `EmailOutbox` – Durable queue of emails with full status history and audit + data. Constraints ensure mutually exclusive sets of render/send error fields + and guard against race conditions. +- `EmailOutboxProcessingMetadata` – Stores the last worker execution timestamp + so we can compute accurate capacity budgets each run. + +## Mutable vs. Immutable States + +Emails can only be edited, paused, retried, or deleted **before** `startedSendingAt` is set. +Once sending begins, the entry becomes read-only. Retrying an email effectively resets its +place in the pipeline & queue, see the Prisma schema for more details. diff --git a/apps/backend/src/app/api/latest/emails/delivery-info/route.tsx b/apps/backend/src/app/api/latest/emails/delivery-info/route.tsx new file mode 100644 index 0000000000..259ea15d9b --- /dev/null +++ b/apps/backend/src/app/api/latest/emails/delivery-info/route.tsx @@ -0,0 +1,65 @@ +import { calculateCapacityRate, getEmailDeliveryStatsForTenancy } from "@/lib/email-delivery-stats"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { adaptSchema, serverOrHigherAuthTypeSchema, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; + +const windows = [ + { key: "hour" as const }, + { key: "day" as const }, + { key: "week" as const }, + { key: "month" as const }, +]; + +export const GET = createSmartRouteHandler({ + metadata: { + summary: "Get email delivery info", + description: "Returns delivery statistics and capacity information for the current tenancy.", + tags: ["Emails"], + }, + request: yupObject({ + auth: yupObject({ + type: serverOrHigherAuthTypeSchema, + tenancy: adaptSchema.defined(), + }).defined(), + method: yupString().oneOf(["GET"]).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + stats: yupObject({ + hour: yupObject({ sent: yupNumber().defined(), bounced: yupNumber().defined(), marked_as_spam: yupNumber().defined() }).defined(), + day: yupObject({ sent: yupNumber().defined(), bounced: yupNumber().defined(), marked_as_spam: yupNumber().defined() }).defined(), + week: yupObject({ sent: yupNumber().defined(), bounced: yupNumber().defined(), marked_as_spam: yupNumber().defined() }).defined(), + month: yupObject({ sent: yupNumber().defined(), bounced: yupNumber().defined(), marked_as_spam: yupNumber().defined() }).defined(), + }).defined(), + capacity: yupObject({ + rate_per_second: yupNumber().defined(), + penalty_factor: yupNumber().defined(), + }).defined(), + }).defined(), + }), + handler: async ({ auth }) => { + const stats = await getEmailDeliveryStatsForTenancy(auth.tenancy.id); + const capacity = calculateCapacityRate(stats); + + return { + statusCode: 200, + bodyType: "json", + body: { + stats: windows.reduce((acc, { key }) => { + const windowStats = stats[key]; + acc[key] = { + sent: windowStats.sent, + bounced: windowStats.bounced, + marked_as_spam: windowStats.markedAsSpam, + }; + return acc; + }, {} as Record), + capacity: { + rate_per_second: capacity.ratePerSecond, + penalty_factor: capacity.penaltyFactor, + }, + }, + }; + }, +}); diff --git a/apps/backend/src/app/api/latest/emails/render-email/route.tsx b/apps/backend/src/app/api/latest/emails/render-email/route.tsx index fef522f6f1..9f197fa64d 100644 --- a/apps/backend/src/app/api/latest/emails/render-email/route.tsx +++ b/apps/backend/src/app/api/latest/emails/render-email/route.tsx @@ -1,4 +1,4 @@ -import { getEmailThemeForTemplate, renderEmailWithTemplate } from "@/lib/email-rendering"; +import { getEmailThemeForThemeId, renderEmailWithTemplate } from "@/lib/email-rendering"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { KnownErrors } from "@stackframe/stack-shared/dist/known-errors"; import { adaptSchema, templateThemeIdSchema, yupNumber, yupObject, yupString, yupUnion } from "@stackframe/stack-shared/dist/schema-fields"; @@ -53,7 +53,7 @@ export const POST = createSmartRouteHandler({ if (typeof body.theme_id === "string" && !themeList.has(body.theme_id)) { throw new StatusError(400, "No theme found with given id"); } - themeSource = getEmailThemeForTemplate(tenancy, body.theme_id); + themeSource = getEmailThemeForThemeId(tenancy, body.theme_id); } let contentSource: string; diff --git a/apps/backend/src/app/api/latest/emails/send-email/route.tsx b/apps/backend/src/app/api/latest/emails/send-email/route.tsx index f3062171cd..42ec4c27c0 100644 --- a/apps/backend/src/app/api/latest/emails/send-email/route.tsx +++ b/apps/backend/src/app/api/latest/emails/send-email/route.tsx @@ -1,16 +1,12 @@ import { getEmailDraft, themeModeToTemplateThemeId } from "@/lib/email-drafts"; -import { createTemplateComponentFromHtml, getEmailThemeForTemplate, renderEmailsWithTemplateBatched } from "@/lib/email-rendering"; -import { getEmailConfig, sendEmail, sendEmailResendBatched } from "@/lib/emails"; -import { getNotificationCategoryByName, hasNotificationEnabled } from "@/lib/notification-categories"; +import { createTemplateComponentFromHtml } from "@/lib/email-rendering"; +import { sendEmailToMany } from "@/lib/emails"; import { getPrismaClientForTenancy } from "@/prisma-client"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; -import { runAsynchronouslyAndWaitUntil } from "@/utils/vercel"; import { KnownErrors } from "@stackframe/stack-shared"; -import { adaptSchema, serverOrHigherAuthTypeSchema, templateThemeIdSchema, yupArray, yupBoolean, yupMixed, yupNumber, yupObject, yupRecord, yupString, yupUnion } from "@stackframe/stack-shared/dist/schema-fields"; -import { getChunks } from "@stackframe/stack-shared/dist/utils/arrays"; +import { adaptSchema, serverOrHigherAuthTypeSchema, templateThemeIdSchema, yupArray, yupBoolean, yupNumber, yupObject, yupRecord, yupString, yupUnion } from "@stackframe/stack-shared/dist/schema-fields"; import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; import { StatusError, throwErr } from "@stackframe/stack-shared/dist/utils/errors"; -import { unsubscribeLinkVerificationCodeHandler } from "../unsubscribe-link/verification-handler"; type UserResult = { user_id: string, @@ -20,11 +16,14 @@ type UserResult = { const bodyBase = yupObject({ user_ids: yupArray(yupString().defined()).optional(), all_users: yupBoolean().oneOf([true]).optional(), - subject: yupString().optional(), - notification_category_name: yupString().optional(), + subject: yupString().optional(), // TODO unused rn; fix this + notification_category_name: yupString().optional(), // TODO unused rn; fix this theme_id: templateThemeIdSchema.nullable().meta({ openapiField: { description: "The theme to use for the email. If not specified, the default theme will be used." } }), + is_high_priority: yupBoolean().optional().meta({ + openapiField: { description: "Marks the email as high priority so it jumps the queue." } + }), }); export const POST = createSmartRouteHandler({ @@ -43,7 +42,7 @@ export const POST = createSmartRouteHandler({ })), bodyBase.concat(yupObject({ template_id: yupString().uuid().defined(), - variables: yupRecord(yupString(), yupMixed()).optional(), + variables: yupRecord(yupString(), yupString().defined()).optional(), })), bodyBase.concat(yupObject({ draft_id: yupString().defined(), @@ -57,7 +56,6 @@ export const POST = createSmartRouteHandler({ body: yupObject({ results: yupArray(yupObject({ user_id: yupString().defined(), - user_email: yupString().optional(), })).defined(), }).defined(), }), @@ -72,192 +70,74 @@ export const POST = createSmartRouteHandler({ throw new KnownErrors.SchemaError("Exactly one of user_ids or all_users must be provided"); } + const isHighPriority = body.is_high_priority ?? false; + const prisma = await getPrismaClientForTenancy(auth.tenancy); - const emailConfig = await getEmailConfig(auth.tenancy); - const defaultNotificationCategory = getNotificationCategoryByName(body.notification_category_name ?? "Transactional") ?? throwErr(400, "Notification category not found with given name"); - let themeSource = getEmailThemeForTemplate(auth.tenancy, body.theme_id); - const variables = "variables" in body ? body.variables : undefined; + + const variables = "variables" in body ? body.variables ?? {} : {}; + const templates = new Map(Object.entries(auth.tenancy.config.emails.templates)); - let templateSource: string; + let tsxSource: string; + let selectedThemeId: string | null | undefined = body.theme_id === false ? null : body.theme_id ?? undefined; // null means empty theme, undefined means use default theme + let createdWith; + if ("template_id" in body) { - templateSource = templates.get(body.template_id)?.tsxSource ?? throwErr(400, "No template found with given template_id"); + const template = templates.get(body.template_id); + if (!template) { + throwErr(400, "No template found with given template_id"); + } + tsxSource = template.tsxSource; + createdWith = { type: "programmatic-call", templateId: body.template_id } as const; } else if ("html" in body) { - templateSource = createTemplateComponentFromHtml(body.html); + tsxSource = createTemplateComponentFromHtml(body.html); + createdWith = { type: "programmatic-call", templateId: null } as const; } else if ("draft_id" in body) { const draft = await getEmailDraft(prisma, auth.tenancy.id, body.draft_id) ?? throwErr(400, "No draft found with given draft_id"); - const theme_id = themeModeToTemplateThemeId(draft.themeMode, draft.themeId); - templateSource = draft.tsxSource; + tsxSource = draft.tsxSource; + createdWith = { type: "draft", draftId: draft.id } as const; + if (body.theme_id === undefined) { - themeSource = getEmailThemeForTemplate(auth.tenancy, theme_id); + const draftThemeId = themeModeToTemplateThemeId(draft.themeMode, draft.themeId); + if (draftThemeId === false) { + selectedThemeId = null; + } else { + selectedThemeId = draftThemeId ?? undefined; + } } } else { throw new KnownErrors.SchemaError("Either template_id, html, or draft_id must be provided"); } - const users = await prisma.projectUser.findMany({ + const requestedUserIds = body.all_users ? (await prisma.projectUser.findMany({ where: { tenancyId: auth.tenancy.id, - projectUserId: { - in: body.user_ids - }, }, - include: { - contactChannels: true, + select: { + projectUserId: true, }, + })).map(user => user.projectUserId) : body.user_ids ?? throwErr("user_ids must be provided if all_users is false"); + + + console.log("Sending email to", requestedUserIds.map(userId => ({ type: "user-primary-email", userId }))); + await sendEmailToMany({ + createdWith: createdWith, + tenancy: auth.tenancy, + recipients: requestedUserIds.map(userId => ({ type: "user-primary-email", userId })), + tsxSource: tsxSource, + extraVariables: variables, + themeId: selectedThemeId === null ? null : (selectedThemeId === undefined ? auth.tenancy.config.emails.selectedThemeId : selectedThemeId), + isHighPriority: isHighPriority, + shouldSkipDeliverabilityCheck: false, + scheduledAt: new Date(), }); - const missingUserIds = body.user_ids?.filter(userId => !users.some(user => user.projectUserId === userId)); - if (missingUserIds && missingUserIds.length > 0) { - throw new KnownErrors.UserIdDoesNotExist(missingUserIds[0]); - } - const userMap = new Map(users.map(user => [user.projectUserId, user])); - const userPrimaryEmails: Map = new Map(); - for (const user of userMap.values()) { - const primaryEmail = user.contactChannels.find((c) => c.isPrimary === "TRUE")?.value; - if (primaryEmail) { - userPrimaryEmails.set(user.projectUserId, primaryEmail); - } - } - - const results: UserResult[] = Array.from(userMap.values()).map((user) => ({ - user_id: user.projectUserId, - user_email: userPrimaryEmails.get(user.projectUserId) ?? user.contactChannels.find((c) => c.isPrimary === "TRUE")?.value, - })); - - const BATCH_SIZE = 100; - - const resolveCategoriesForUsers = async (usersWithPrimary: typeof users) => { - const currentCategories = new Map>(); - if (!("html" in body)) { - const firstPassInputs = usersWithPrimary.map((user) => ({ - user: { displayName: user.displayName }, - project: { displayName: auth.tenancy.project.display_name }, - variables, - })); - - const chunks = getChunks(firstPassInputs, BATCH_SIZE); - const userChunks = getChunks(usersWithPrimary, BATCH_SIZE); - for (let i = 0; i < chunks.length; i++) { - const chunk = chunks[i]; - const correspondingUsers = userChunks[i]; - const rendered = await renderEmailsWithTemplateBatched(templateSource, themeSource, chunk); - if (rendered.status === "error") { - continue; - } - const outputs = rendered.data; - for (let j = 0; j < outputs.length; j++) { - const output = outputs[j]; - const user = correspondingUsers[j]; - const category = getNotificationCategoryByName(output.notificationCategory ?? ""); - currentCategories.set(user.projectUserId, category); - } - } - } else { - for (const user of usersWithPrimary) { - currentCategories.set(user.projectUserId, defaultNotificationCategory); - } - } - return currentCategories; - }; - const getAllowedUsersWithUnsub = async (usersWithPrimary: typeof users, currentCategories: Map>) => { - const allowed = await Promise.all(usersWithPrimary.map(async (user) => { - const category = currentCategories.get(user.projectUserId) ?? defaultNotificationCategory; - const enabled = await hasNotificationEnabled(auth.tenancy, user.projectUserId, category.id); - return enabled ? { user, category } : null; - })).then(r => r.filter((x): x is { user: typeof users[number], category: NonNullable> } => Boolean(x))); - const unsubLinks = new Map(); - await Promise.all(allowed.map(async ({ user, category }) => { - if (!category.can_disable) { - unsubLinks.set(user.projectUserId, undefined); - return; - } - const { code } = await unsubscribeLinkVerificationCodeHandler.createCode({ - tenancy: auth.tenancy, - method: {}, - data: { - user_id: user.projectUserId, - notification_category_id: category.id, - }, - callbackUrl: undefined - }); - const unsubUrl = new URL(getEnvVariable("NEXT_PUBLIC_STACK_API_URL")); - unsubUrl.pathname = "/api/v1/emails/unsubscribe-link"; - unsubUrl.searchParams.set("code", code); - unsubLinks.set(user.projectUserId, unsubUrl.toString()); - })); - return { allowed, unsubLinks }; - }; - - const renderAndSendBatches = async (finalUsers: typeof users, unsubLinks: Map) => { - const finalInputs = finalUsers.map((user) => ({ - user: { displayName: user.displayName }, - project: { displayName: auth.tenancy.project.display_name }, - variables, - unsubscribeLink: unsubLinks.get(user.projectUserId), - })); - - const inputChunks = getChunks(finalInputs, BATCH_SIZE); - const userChunks = getChunks(finalUsers, BATCH_SIZE); - - for (let i = 0; i < inputChunks.length; i++) { - const chunk = inputChunks[i]; - const correspondingUsers = userChunks[i]; - const rendered = await renderEmailsWithTemplateBatched(templateSource, themeSource, chunk); - if (rendered.status === "error") { - continue; - } - const outputs = rendered.data; - const emailOptions = outputs.map((output, idx) => { - const user = correspondingUsers[idx]; - const email = userPrimaryEmails.get(user.projectUserId); - if (!email) return null; - return { - tenancyId: auth.tenancy.id, - emailConfig, - to: email, - subject: body.subject ?? output.subject ?? "", - html: output.html, - text: output.text, - }; - }).filter((option): option is NonNullable => Boolean(option)); - - if (emailConfig.host === "smtp.resend.com") { - await sendEmailResendBatched(emailConfig.password, emailOptions); - } else { - await Promise.allSettled(emailOptions.map(option => sendEmail(option))); - } - } - }; - - runAsynchronouslyAndWaitUntil((async () => { - const usersArray = Array.from(userMap.values()); - - const usersWithPrimary = usersArray.filter(u => userPrimaryEmails.has(u.projectUserId)); - const currentCategories = await resolveCategoriesForUsers(usersWithPrimary); - const { allowed, unsubLinks } = await getAllowedUsersWithUnsub(usersWithPrimary, currentCategories); - const finalUsers = allowed.map(({ user }) => user); - await renderAndSendBatches(finalUsers, unsubLinks); - - if ("draft_id" in body) { - await prisma.emailDraft.update({ - where: { - tenancyId_id: { - tenancyId: auth.tenancy.id, - id: body.draft_id, - }, - }, - data: { sentAt: new Date() }, - }); - } - })()); - - if ("draft_id" in body) { + if (createdWith.type === "draft") { await prisma.emailDraft.update({ where: { tenancyId_id: { tenancyId: auth.tenancy.id, - id: body.draft_id, + id: createdWith.draftId, }, }, data: { sentAt: new Date() }, @@ -267,7 +147,9 @@ export const POST = createSmartRouteHandler({ return { statusCode: 200, bodyType: 'json', - body: { results }, + body: { + results: requestedUserIds.map(userId => ({ user_id: userId })), + }, }; }, }); diff --git a/apps/backend/src/app/api/latest/integrations/credential-scanning/revoke/route.tsx b/apps/backend/src/app/api/latest/integrations/credential-scanning/revoke/route.tsx index 16bf5c56a6..bec9c3784c 100644 --- a/apps/backend/src/app/api/latest/integrations/credential-scanning/revoke/route.tsx +++ b/apps/backend/src/app/api/latest/integrations/credential-scanning/revoke/route.tsx @@ -1,4 +1,3 @@ -import { getSharedEmailConfig, sendEmail } from "@/lib/emails"; import { listPermissions } from "@/lib/permissions"; import { getTenancy } from "@/lib/tenancies"; import { getPrismaClientForTenancy, globalPrismaClient, retryTransaction } from "@/prisma-client"; @@ -184,18 +183,8 @@ export const POST = createSmartRouteHandler({ `; - const emailConfig = await getSharedEmailConfig("Stack Auth"); - // Send email notifications - for (const email of affectedEmails) { - await sendEmail({ - tenancyId: updatedApiKey.tenancyId, - emailConfig, - to: email, - subject, - html: htmlContent, - }); - } + throw new StackAssertionError("Credential scanning email is currently disabled!"); return { statusCode: 200, diff --git a/apps/backend/src/app/api/latest/internal/email-queue-step/route.tsx b/apps/backend/src/app/api/latest/internal/email-queue-step/route.tsx new file mode 100644 index 0000000000..aa8d84f8a3 --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/email-queue-step/route.tsx @@ -0,0 +1,40 @@ +import { runEmailQueueStep } from "@/lib/email-queue-step"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; +import { wait } from "@stackframe/stack-shared/dist/utils/promises"; + +export const GET = createSmartRouteHandler({ + metadata: { + summary: "Process email queue step", + description: "Internal endpoint invoked by Vercel Cron to advance the email sending pipeline.", + tags: ["Emails"], + hidden: true, + }, + request: yupObject({ + auth: yupObject({}).nullable().optional(), + method: yupString().oneOf(["GET"]).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + ok: yupBoolean().defined(), + }).defined(), + }), + handler: async (_req, fullReq) => { + const startTime = performance.now(); + + while (performance.now() - startTime < 2 * 60 * 1000) { + await runEmailQueueStep(); + await wait(1000); + } + + return { + statusCode: 200, + bodyType: "json", + body: { + ok: true, + }, + }; + }, +}); diff --git a/apps/backend/src/app/api/latest/internal/emails/crud.tsx b/apps/backend/src/app/api/latest/internal/emails/crud.tsx index aa5be457d2..fe1e4c7819 100644 --- a/apps/backend/src/app/api/latest/internal/emails/crud.tsx +++ b/apps/backend/src/app/api/latest/internal/emails/crud.tsx @@ -1,27 +1,29 @@ import { getPrismaClientForTenancy } from "@/prisma-client"; import { createCrudHandlers } from "@/route-handlers/crud-handler"; -import { SentEmail } from "@prisma/client"; +import { EmailOutbox } from "@prisma/client"; import { InternalEmailsCrud, internalEmailsCrud } from "@stackframe/stack-shared/dist/interface/crud/emails"; import { yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; import { createLazyProxy } from "@stackframe/stack-shared/dist/utils/proxies"; -function prismaModelToCrud(prismaModel: SentEmail): InternalEmailsCrud["Admin"]["Read"] { - const senderConfig = prismaModel.senderConfig as any; +function prismaModelToCrud(prismaModel: EmailOutbox): InternalEmailsCrud["Admin"]["Read"] { + const recipient = prismaModel.to as any; + let to: string[] = []; + if (recipient?.type === 'user-primary-email') { + to = [`User ID: ${recipient.userId}`]; + } else if (recipient?.type === 'user-custom-emails' || recipient?.type === 'custom-emails') { + to = Array.isArray(recipient.emails) ? recipient.emails : []; + } + + let error: string | null = null; + if (prismaModel.renderErrorExternalMessage) error = `Render error: ${prismaModel.renderErrorExternalMessage}`; + else if (prismaModel.sendServerErrorExternalMessage) error = `Send error: ${prismaModel.sendServerErrorExternalMessage}`; return { id: prismaModel.id, - subject: prismaModel.subject, - sent_at_millis: prismaModel.createdAt.getTime(), - to: prismaModel.to, - sender_config: { - type: senderConfig.type, - host: senderConfig.host, - port: senderConfig.port, - username: senderConfig.username, - sender_name: senderConfig.senderName, - sender_email: senderConfig.senderEmail, - }, - error: prismaModel.error, + subject: prismaModel.renderedSubject ?? "", + sent_at_millis: (prismaModel.finishedSendingAt ?? prismaModel.createdAt).getTime(), + to, + error: error, }; } @@ -33,7 +35,7 @@ export const internalEmailsCrudHandlers = createLazyProxy(() => createCrudHandle onList: async ({ auth }) => { const prisma = await getPrismaClientForTenancy(auth.tenancy); - const emails = await prisma.sentEmail.findMany({ + const emails = await prisma.emailOutbox.findMany({ where: { tenancyId: auth.tenancy.id, }, diff --git a/apps/backend/src/app/api/latest/internal/failed-emails-digest/crud.tsx b/apps/backend/src/app/api/latest/internal/failed-emails-digest/crud.tsx index 8bb9f3eb0d..fb9ec23824 100644 --- a/apps/backend/src/app/api/latest/internal/failed-emails-digest/crud.tsx +++ b/apps/backend/src/app/api/latest/internal/failed-emails-digest/crud.tsx @@ -1,10 +1,11 @@ +import { EmailOutboxRecipient } from "@/lib/emails"; import { globalPrismaClient } from "@/prisma-client"; type FailedEmailsQueryResult = { tenancyId: string, projectId: string, - to: string[], - subject: string, + to: EmailOutboxRecipient, + subject: string | null, contactEmail: string, } @@ -18,13 +19,13 @@ export const getFailedEmailsByTenancy = async (after: Date) => { // Only email digest for hosted DB is supported for now. const result = await globalPrismaClient.$queryRaw>` SELECT - se."tenancyId", + eo."tenancyId", t."projectId", - se."to", - se."subject", + eo."to", + eo."renderedSubject" as "subject", cc."value" as "contactEmail" - FROM "SentEmail" se - INNER JOIN "Tenancy" t ON se."tenancyId" = t.id + FROM "EmailOutbox" eo + INNER JOIN "Tenancy" t ON eo."tenancyId" = t.id INNER JOIN "Project" p ON t."projectId" = p.id LEFT JOIN "ProjectUser" pu ON pu."mirroredProjectId" = 'internal' AND pu."mirroredBranchId" = 'main' @@ -34,8 +35,8 @@ export const getFailedEmailsByTenancy = async (after: Date) => { INNER JOIN "ContactChannel" cc ON tm."projectUserId" = cc."projectUserId" AND cc."isPrimary" = 'TRUE' AND cc."type" = 'EMAIL' - WHERE se."error" IS NOT NULL - AND se."createdAt" >= ${after} + WHERE eo."simpleStatus" = 'ERROR'::"EmailOutboxSimpleStatus" + AND eo."createdAt" >= ${after} `; const failedEmailsByTenancy = new Map(); @@ -45,7 +46,25 @@ export const getFailedEmailsByTenancy = async (after: Date) => { tenantOwnerEmails: [], projectId: failedEmail.projectId }; - failedEmails.emails.push({ subject: failedEmail.subject, to: failedEmail.to }); + + let to: string[] = []; + const recipient = failedEmail.to; + switch (recipient.type) { + case 'user-primary-email': { + to = [`User ID: ${recipient.userId}`]; + break; + } + case 'user-custom-emails': { + to = Array.isArray(recipient.emails) ? recipient.emails : []; + break; + } + case 'custom-emails': { + to = Array.isArray(recipient.emails) ? recipient.emails : []; + break; + } + } + + failedEmails.emails.push({ subject: failedEmail.subject ?? "(No Subject)", to }); failedEmails.tenantOwnerEmails.push(failedEmail.contactEmail); failedEmailsByTenancy.set(failedEmail.tenancyId, failedEmails); } diff --git a/apps/backend/src/app/api/latest/internal/failed-emails-digest/route.ts b/apps/backend/src/app/api/latest/internal/failed-emails-digest/route.ts index b460ede034..0ed66f3eba 100644 --- a/apps/backend/src/app/api/latest/internal/failed-emails-digest/route.ts +++ b/apps/backend/src/app/api/latest/internal/failed-emails-digest/route.ts @@ -1,9 +1,9 @@ -import { getSharedEmailConfig, sendEmail } from "@/lib/emails"; +import { getSharedEmailConfig } from "@/lib/emails"; import { DEFAULT_BRANCH_ID, getSoleTenancyFromProjectBranch } from "@/lib/tenancies"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { yupArray, yupBoolean, yupNumber, yupObject, yupString, yupTuple } from "@stackframe/stack-shared/dist/schema-fields"; import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; -import { StatusError, captureError } from "@stackframe/stack-shared/dist/utils/errors"; +import { StackAssertionError, StatusError, captureError } from "@stackframe/stack-shared/dist/utils/errors"; import { escapeHtml } from "@stackframe/stack-shared/dist/utils/html"; import { getFailedEmailsByTenancy } from "./crud"; @@ -69,13 +69,7 @@ export const POST = createSmartRouteHandler({ `; if (query.dry_run !== "true") { try { - await sendEmail({ - tenancyId: internalTenancy.id, - emailConfig, - to: failedEmailsBatch.tenantOwnerEmails, - subject: "Failed emails digest", - html: emailHtml, - }); + throw new StackAssertionError("Failed emails digest is currently disabled!"); } catch (error) { anyDigestsFailedToSend = true; captureError("send-failed-emails-digest", error); diff --git a/apps/backend/src/app/api/latest/internal/send-sign-in-invitation/route.tsx b/apps/backend/src/app/api/latest/internal/send-sign-in-invitation/route.tsx index 697d3cab6c..57a6735935 100644 --- a/apps/backend/src/app/api/latest/internal/send-sign-in-invitation/route.tsx +++ b/apps/backend/src/app/api/latest/internal/send-sign-in-invitation/route.tsx @@ -1,4 +1,4 @@ -import { sendEmailFromTemplate } from "@/lib/emails"; +import { sendEmailFromDefaultTemplate } from "@/lib/emails"; import { validateRedirectUrl } from "@/lib/redirect-urls"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { KnownErrors } from "@stackframe/stack-shared"; @@ -32,7 +32,7 @@ export const POST = createSmartRouteHandler({ throw new KnownErrors.RedirectUrlNotWhitelisted(); } - await sendEmailFromTemplate({ + await sendEmailFromDefaultTemplate({ email: body.email, tenancy: auth.tenancy, user: null, @@ -41,6 +41,7 @@ export const POST = createSmartRouteHandler({ signInInvitationLink: body.callback_url, teamDisplayName: auth.tenancy.project.display_name, }, + shouldSkipDeliverabilityCheck: true, }); return { diff --git a/apps/backend/src/app/api/latest/internal/send-test-email/route.tsx b/apps/backend/src/app/api/latest/internal/send-test-email/route.tsx index 2cb27259b2..ccbbe2b5be 100644 --- a/apps/backend/src/app/api/latest/internal/send-test-email/route.tsx +++ b/apps/backend/src/app/api/latest/internal/send-test-email/route.tsx @@ -1,4 +1,4 @@ -import { isSecureEmailPort, sendEmailWithoutRetries } from "@/lib/emails"; +import { isSecureEmailPort, lowLevelSendEmailDirectWithoutRetries } from "@/lib/emails-low-level"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import * as schemaFields from "@stackframe/stack-shared/dist/schema-fields"; import { adaptSchema, adminAuthTypeSchema, emailSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; @@ -37,7 +37,7 @@ export const POST = createSmartRouteHandler({ }).defined(), }), handler: async ({ body, auth }) => { - const resultOuter = await timeout(sendEmailWithoutRetries({ + const resultOuter = await timeout(lowLevelSendEmailDirectWithoutRetries({ tenancyId: auth.tenancy.id, emailConfig: { type: 'standard', diff --git a/apps/backend/src/app/api/latest/team-invitations/accept/verification-code-handler.tsx b/apps/backend/src/app/api/latest/team-invitations/accept/verification-code-handler.tsx index 2ecceb4bc9..f309ac92c7 100644 --- a/apps/backend/src/app/api/latest/team-invitations/accept/verification-code-handler.tsx +++ b/apps/backend/src/app/api/latest/team-invitations/accept/verification-code-handler.tsx @@ -1,5 +1,5 @@ import { teamMembershipsCrudHandlers } from "@/app/api/latest/team-memberships/crud"; -import { sendEmailFromTemplate } from "@/lib/emails"; +import { sendEmailFromDefaultTemplate } from "@/lib/emails"; import { getItemQuantityForCustomer } from "@/lib/payments"; import { getSoleTenancyFromProjectBranch } from "@/lib/tenancies"; import { getPrismaClientForTenancy } from "@/prisma-client"; @@ -54,7 +54,7 @@ export const teamInvitationCodeHandler = createVerificationCodeHandler({ team_id: createOptions.data.team_id, }); - await sendEmailFromTemplate({ + await sendEmailFromDefaultTemplate({ tenancy: await getSoleTenancyFromProjectBranch(createOptions.project, createOptions.branchId), user: null, email: createOptions.method.email, @@ -63,6 +63,7 @@ export const teamInvitationCodeHandler = createVerificationCodeHandler({ teamInvitationLink: codeObj.link.toString(), teamDisplayName: team.display_name, }, + shouldSkipDeliverabilityCheck: true, }); return codeObj; diff --git a/apps/backend/src/lib/email-delivery-stats.tsx b/apps/backend/src/lib/email-delivery-stats.tsx new file mode 100644 index 0000000000..9b5a5a74a9 --- /dev/null +++ b/apps/backend/src/lib/email-delivery-stats.tsx @@ -0,0 +1,101 @@ +import { globalPrismaClient, PrismaClientTransaction, RawQuery, rawQuery } from "@/prisma-client"; +import { Prisma } from "@prisma/client"; + +export type EmailDeliveryWindowStats = { + sent: number, + bounced: number, + markedAsSpam: number, +}; + +export type EmailDeliveryStats = { + hour: EmailDeliveryWindowStats, + day: EmailDeliveryWindowStats, + week: EmailDeliveryWindowStats, + month: EmailDeliveryWindowStats, +}; + +export function calculatePenaltyFactor(sent: number, bounced: number, spam: number): number { + if (sent === 0) { + return 1; + } + const failures = bounced + 50 * spam; + const failureRate = failures / sent; + return Math.max(0.1, Math.min(1, 1 - failureRate)); +} + +export function calculateCapacityRate(stats: EmailDeliveryStats) { + const penaltyFactor = Math.min( + calculatePenaltyFactor(stats.week.sent, stats.week.bounced, stats.week.markedAsSpam), + calculatePenaltyFactor(stats.day.sent, stats.day.bounced, stats.day.markedAsSpam), + calculatePenaltyFactor(stats.hour.sent, stats.hour.bounced, stats.hour.markedAsSpam) + ); + const hourlyBaseline = 200 * 24 * 7 + (8 * 30 * 24 * stats.month.sent); + const adjusted = Math.max(hourlyBaseline * penaltyFactor, 7 * 24 * 60); + const ratePerSecond = adjusted / (7 * 60 * 60); + return { ratePerSecond, penaltyFactor }; +} + +const deliveryStatsQuery = (tenancyId: string): RawQuery => ({ + supportedPrismaClients: ["global"], + sql: Prisma.sql` + SELECT + SUM(CASE WHEN "finishedSendingAt" >= NOW() - INTERVAL '1 hour' AND "sendServerErrorInternalMessage" IS NULL AND "skippedReason" IS NULL THEN 1 ELSE 0 END)::bigint AS sent_last_hour, + SUM(CASE WHEN "finishedSendingAt" >= NOW() - INTERVAL '1 day' AND "sendServerErrorInternalMessage" IS NULL AND "skippedReason" IS NULL THEN 1 ELSE 0 END)::bigint AS sent_last_day, + SUM(CASE WHEN "finishedSendingAt" >= NOW() - INTERVAL '1 week' AND "sendServerErrorInternalMessage" IS NULL AND "skippedReason" IS NULL THEN 1 ELSE 0 END)::bigint AS sent_last_week, + SUM(CASE WHEN "finishedSendingAt" >= NOW() - INTERVAL '1 month' AND "sendServerErrorInternalMessage" IS NULL AND "skippedReason" IS NULL THEN 1 ELSE 0 END)::bigint AS sent_last_month, + SUM(CASE WHEN "finishedSendingAt" >= NOW() - INTERVAL '1 hour' AND "bouncedAt" IS NOT NULL THEN 1 ELSE 0 END)::bigint AS bounced_last_hour, + SUM(CASE WHEN "finishedSendingAt" >= NOW() - INTERVAL '1 day' AND "bouncedAt" IS NOT NULL THEN 1 ELSE 0 END)::bigint AS bounced_last_day, + SUM(CASE WHEN "finishedSendingAt" >= NOW() - INTERVAL '1 week' AND "bouncedAt" IS NOT NULL THEN 1 ELSE 0 END)::bigint AS bounced_last_week, + SUM(CASE WHEN "finishedSendingAt" >= NOW() - INTERVAL '1 month' AND "bouncedAt" IS NOT NULL THEN 1 ELSE 0 END)::bigint AS bounced_last_month, + SUM(CASE WHEN "finishedSendingAt" >= NOW() - INTERVAL '1 hour' AND "markedAsSpamAt" IS NOT NULL THEN 1 ELSE 0 END)::bigint AS spam_last_hour, + SUM(CASE WHEN "finishedSendingAt" >= NOW() - INTERVAL '1 day' AND "markedAsSpamAt" IS NOT NULL THEN 1 ELSE 0 END)::bigint AS spam_last_day, + SUM(CASE WHEN "finishedSendingAt" >= NOW() - INTERVAL '1 week' AND "markedAsSpamAt" IS NOT NULL THEN 1 ELSE 0 END)::bigint AS spam_last_week, + SUM(CASE WHEN "finishedSendingAt" >= NOW() - INTERVAL '1 month' AND "markedAsSpamAt" IS NOT NULL THEN 1 ELSE 0 END)::bigint AS spam_last_month + FROM "EmailOutbox" + WHERE "tenancyId" = ${tenancyId}::uuid + `, + postProcess: (rows) => { + const row = rows[0] ?? { + sent_last_hour: 0n, + sent_last_day: 0n, + sent_last_week: 0n, + sent_last_month: 0n, + bounced_last_hour: 0n, + bounced_last_day: 0n, + bounced_last_week: 0n, + bounced_last_month: 0n, + spam_last_hour: 0n, + spam_last_day: 0n, + spam_last_week: 0n, + spam_last_month: 0n, + }; + const toNumber = (value: unknown) => Number(value ?? 0); + return { + hour: { + sent: toNumber(row.sent_last_hour), + bounced: toNumber(row.bounced_last_hour), + markedAsSpam: toNumber(row.spam_last_hour), + }, + day: { + sent: toNumber(row.sent_last_day), + bounced: toNumber(row.bounced_last_day), + markedAsSpam: toNumber(row.spam_last_day), + }, + week: { + sent: toNumber(row.sent_last_week), + bounced: toNumber(row.bounced_last_week), + markedAsSpam: toNumber(row.spam_last_week), + }, + month: { + sent: toNumber(row.sent_last_month), + bounced: toNumber(row.bounced_last_month), + markedAsSpam: toNumber(row.spam_last_month), + }, + }; + }, +}); + +export async function getEmailDeliveryStatsForTenancy(tenancyId: string, tx?: PrismaClientTransaction): Promise { + const client = tx ?? globalPrismaClient; + return await rawQuery(client, deliveryStatsQuery(tenancyId)); +} diff --git a/apps/backend/src/lib/email-queue-step.tsx b/apps/backend/src/lib/email-queue-step.tsx new file mode 100644 index 0000000000..cde61ae341 --- /dev/null +++ b/apps/backend/src/lib/email-queue-step.tsx @@ -0,0 +1,619 @@ +import { calculateCapacityRate, getEmailDeliveryStatsForTenancy } from "@/lib/email-delivery-stats"; +import { enqueueEmailQueueStep } from "@/lib/email-queue"; +import { getEmailThemeForThemeId, renderEmailsForTenancyBatched } from "@/lib/email-rendering"; +import { EmailOutboxRecipient, getEmailConfig, } from "@/lib/emails"; +import { generateUnsubscribeLink, getNotificationCategoryById } from "@/lib/notification-categories"; +import { getTenancy, Tenancy } from "@/lib/tenancies"; +import { getPrismaClientForTenancy, globalPrismaClient, PrismaClientTransaction } from "@/prisma-client"; +import { withTraceSpan } from "@/utils/telemetry"; +import { allPromisesAndWaitUntilEach } from "@/utils/vercel"; +import { EmailOutbox, EmailOutboxSkippedReason, Prisma } from "@prisma/client"; +import { groupBy } from "@stackframe/stack-shared/dist/utils/arrays"; +import { captureError, errorToNiceString, StackAssertionError, throwErr } from "@stackframe/stack-shared/dist/utils/errors"; +import { Json } from "@stackframe/stack-shared/dist/utils/json"; +import { filterUndefined } from "@stackframe/stack-shared/dist/utils/objects"; +import { Result } from "@stackframe/stack-shared/dist/utils/results"; +import { randomUUID } from "node:crypto"; +import { lowLevelSendEmailDirectViaProvider } from "./emails-low-level"; + +const MAX_RENDER_BATCH = 50; + +type TenancySendBatch = { + tenancyId: string, + rows: EmailOutbox[], + capacityRatePerSecond: number, +}; + +// note: there is no locking surrounding this function, so it may run multiple times concurrently. It needs to deal with that. +export const runEmailQueueStep = withTraceSpan("runEmailQueueStep", async () => { + await withTraceSpan("runEmailQueueStep-enqueueEmailQueueStep", enqueueEmailQueueStep)(); + + const workerId = randomUUID(); + + const deltaSeconds = await withTraceSpan("runEmailQueueStep-updateLastExecutionTime", updateLastExecutionTime)(); + if (deltaSeconds <= 0) { + return; + } + + + const pendingRender = await withTraceSpan("runEmailQueueStep-claimEmailsForRendering", claimEmailsForRendering)(workerId); + await withTraceSpan("runEmailQueueStep-renderEmails", renderEmails)(workerId, pendingRender); + await withTraceSpan("runEmailQueueStep-retryEmailsStuckInRendering", retryEmailsStuckInRendering)(); + + await withTraceSpan("runEmailQueueStep-queueReadyEmails", queueReadyEmails)(); + + const sendPlan = await withTraceSpan("runEmailQueueStep-prepareSendPlan", prepareSendPlan)(deltaSeconds); + await withTraceSpan("runEmailQueueStep-processSendPlan", processSendPlan)(sendPlan); +}); + +async function retryEmailsStuckInRendering(): Promise { + const res = await globalPrismaClient.emailOutbox.updateManyAndReturn({ + where: { + startedRenderingAt: { + lte: new Date(Date.now() - 1000 * 60 * 20), + }, + finishedRenderingAt: null, + }, + data: { + renderedByWorkerId: null, + startedRenderingAt: null, + }, + }); + if (res.length > 0) { + captureError("email-queue-step-stuck-in-rendering", new StackAssertionError("Emails stuck in rendering! This should never happen. Resetting them to be re-rendered.", { + emails: res.map(e => e.id), + })); + } +} +async function updateLastExecutionTime(): Promise { + const key = "EMAIL_QUEUE_METADATA_KEY"; + + const [{ delta }] = await globalPrismaClient.$queryRaw<{ delta: number }[]>` + WITH now_ts AS ( + SELECT NOW() AS now + ), + existing AS ( + SELECT "lastExecutedAt" + FROM "EmailOutboxProcessingMetadata" + WHERE "key" = ${key} + ), + action AS ( + INSERT INTO "EmailOutboxProcessingMetadata" ("key", "lastExecutedAt", "updatedAt") + VALUES (${key}, (SELECT now FROM now_ts), (SELECT now FROM now_ts)) + ON CONFLICT ("key") DO UPDATE SET + "updatedAt" = (SELECT now FROM now_ts), + "lastExecutedAt" = CASE + WHEN "EmailOutboxProcessingMetadata"."lastExecutedAt" IS NULL + OR "EmailOutboxProcessingMetadata"."lastExecutedAt" < (SELECT now FROM now_ts) + THEN (SELECT now FROM now_ts) + ELSE "EmailOutboxProcessingMetadata"."lastExecutedAt" + END + RETURNING "lastExecutedAt" + ) + SELECT + CASE + WHEN (SELECT "lastExecutedAt" FROM existing) IS NULL THEN 0 + WHEN (SELECT "lastExecutedAt" FROM action) = (SELECT "lastExecutedAt" FROM existing) THEN 0 + ELSE EXTRACT(EPOCH FROM ( + (SELECT "lastExecutedAt" FROM action) - + (SELECT "lastExecutedAt" FROM existing) + )) + END AS delta; + `; + + return delta; +} + +async function claimEmailsForRendering(workerId: string): Promise { + return await globalPrismaClient.$queryRaw(Prisma.sql` + WITH selected AS ( + SELECT "tenancyId", "id" + FROM "EmailOutbox" + WHERE "renderedByWorkerId" IS NULL + AND "isPaused" = FALSE + ORDER BY "createdAt" ASC + LIMIT ${MAX_RENDER_BATCH} + FOR UPDATE SKIP LOCKED + ) + UPDATE "EmailOutbox" AS e + SET + "renderedByWorkerId" = ${workerId}::uuid, + "startedRenderingAt" = NOW() + FROM selected + WHERE e."tenancyId" = selected."tenancyId" AND e."id" = selected."id" + RETURNING e.*; + `); +} + +async function renderEmails(workerId: string, rows: EmailOutbox[]): Promise { + const rowsByTenancy = groupBy(rows, outbox => outbox.tenancyId); + + for (const [tenancyId, group] of rowsByTenancy.entries()) { + try { + await renderTenancyEmails(workerId, tenancyId, group); + } catch (error) { + captureError("email-queue-step-rendering-error", error); + } + } +} + +async function renderTenancyEmails(workerId: string, tenancyId: string, group: EmailOutbox[]): Promise { + const tenancy = await getTenancy(tenancyId) ?? throwErr("Tenancy not found in renderTenancyEmails? Was the tenancy deletion not cascaded?"); + + const prisma = await getPrismaClientForTenancy(tenancy); + const userRecipientRows = group.filter((row) => { + const recipient = deserializeRecipient(row.to as Json); + return recipient.type !== "custom-emails"; + }); + + const userIds = new Set(); + for (const row of userRecipientRows) { + const recipient = deserializeRecipient(row.to as Json); + if ("userId" in recipient) { + userIds.add(recipient.userId); + } + } + + const users = userIds.size > 0 ? await prisma.projectUser.findMany({ + where: { + tenancyId: tenancy.id, + projectUserId: { in: Array.from(userIds) }, + }, + include: { + contactChannels: true, + }, + }) : []; + + const userMap = new Map(users.map(user => [user.projectUserId, user])); + + const requests = await Promise.all(group.map(async (row) => { + const themeSource = getEmailThemeForThemeId(tenancy, row.themeId ?? false); + + const recipient = deserializeRecipient(row.to as Json); + let userDisplayName: string | null = null; + let unsubscribeLink: string | undefined; + if ("userId" in recipient) { + const user = userMap.get(recipient.userId); + userDisplayName = user?.displayName ?? null; + if (row.renderedNotificationCategoryId) { + const category = getNotificationCategoryById(row.renderedNotificationCategoryId); + if (category?.can_disable) { + const unsubscribeResult = await Result.fromPromise(generateUnsubscribeLink(tenancy, recipient.userId, row.renderedNotificationCategoryId)); + if (unsubscribeResult.status === "ok") { + unsubscribeLink = unsubscribeResult.data; + } else { + captureError("generate-unsubscribe-link", unsubscribeResult.error); + } + } + } + } + + return { + templateSource: row.tsxSource, + themeSource, + input: { + user: { displayName: userDisplayName }, + project: { displayName: tenancy.project.display_name }, + variables: filterUndefined({ + projectDisplayName: tenancy.project.display_name, + userDisplayName: userDisplayName, + ...filterUndefined((row.extraRenderVariables ?? {}) as Record), + }), + unsubscribeLink, + }, + }; + })); + + const renderResult = await renderEmailsForTenancyBatched(requests); + if (renderResult.status === "error") { + captureError("email-rendering-failed", renderResult.error); + for (const row of group) { + await globalPrismaClient.emailOutbox.updateMany({ + where: { + tenancyId, + id: row.id, + renderedByWorkerId: workerId, + }, + data: { + renderErrorExternalMessage: "An error occurred while rendering the email. Make sure the template/draft is valid and the theme is set correctly.", + renderErrorExternalDetails: {}, + renderErrorInternalMessage: renderResult.error, + renderErrorInternalDetails: { error: renderResult.error }, + finishedRenderingAt: new Date(), + }, + }); + } + return; + } + + const outputs = renderResult.data; + for (let index = 0; index < group.length; index++) { + const row = group[index]; + const output = outputs[index]; + await globalPrismaClient.emailOutbox.updateMany({ + where: { + tenancyId, + id: row.id, + renderedByWorkerId: workerId, + }, + data: { + renderedHtml: output.html, + renderedText: output.text, + renderedSubject: output.subject ?? "", + renderedNotificationCategoryId: output.notificationCategory, + renderedIsTransactional: output.notificationCategory === "transactional", // TODO this should use smarter logic for notification category handling + renderErrorExternalMessage: null, + renderErrorExternalDetails: Prisma.DbNull, + renderErrorInternalMessage: null, + renderErrorInternalDetails: Prisma.DbNull, + finishedRenderingAt: new Date(), + }, + }); + } +} + +async function queueReadyEmails(): Promise { + await globalPrismaClient.$executeRaw` + UPDATE "EmailOutbox" + SET "isQueued" = TRUE + WHERE "isQueued" = FALSE + AND "isPaused" = FALSE + AND "finishedRenderingAt" IS NOT NULL + AND "renderedHtml" IS NOT NULL + AND "scheduledAt" <= NOW() + `; +} + +async function prepareSendPlan(deltaSeconds: number): Promise { + const tenancyIds = await globalPrismaClient.emailOutbox.findMany({ + where: { + isQueued: true, + isPaused: false, + startedSendingAt: null, + }, + distinct: ["tenancyId"], + select: { tenancyId: true }, + }); + + const plan: TenancySendBatch[] = []; + for (const entry of tenancyIds) { + const stats = await getEmailDeliveryStatsForTenancy(entry.tenancyId); + const capacity = calculateCapacityRate(stats); + const quota = stochasticQuota(capacity.ratePerSecond * deltaSeconds); + if (quota <= 0) continue; + const rows = await claimEmailsForSending(globalPrismaClient, entry.tenancyId, quota); + if (rows.length === 0) continue; + plan.push({ tenancyId: entry.tenancyId, rows, capacityRatePerSecond: capacity.ratePerSecond }); + } + return plan; +} + +function stochasticQuota(value: number): number { + const base = Math.floor(value); + const fractional = value - base; + return base + (Math.random() < fractional ? 1 : 0); +} + +async function claimEmailsForSending(tx: PrismaClientTransaction, tenancyId: string, limit: number): Promise { + return await tx.$queryRaw(Prisma.sql` + WITH selected AS ( + SELECT "tenancyId", "id" + FROM "EmailOutbox" + WHERE "tenancyId" = ${tenancyId}::uuid + AND "isQueued" = TRUE + AND "isPaused" = FALSE + AND "finishedRenderingAt" IS NOT NULL + AND "startedSendingAt" IS NULL + ORDER BY "priority" DESC, "scheduledAt" ASC, "createdAt" ASC + LIMIT ${limit} + FOR UPDATE SKIP LOCKED + ) + UPDATE "EmailOutbox" AS e + SET "startedSendingAt" = NOW() + FROM selected + WHERE e."tenancyId" = selected."tenancyId" AND e."id" = selected."id" + RETURNING e.*; + `); +} + +async function processSendPlan(plan: TenancySendBatch[]): Promise { + for (const batch of plan) { + try { + await processTenancyBatch(batch); + } catch (error) { + captureError("email-queue-step-sending-error", error); + } + } +} + +type ProjectUserWithContacts = Prisma.ProjectUserGetPayload<{ include: { contactChannels: true } }>; + +type TenancyProcessingContext = { + tenancy: Tenancy, + prisma: Awaited>, + emailConfig: Awaited>, + userMap: Map, + notificationPreferences: Map>, +}; + +async function processTenancyBatch(batch: TenancySendBatch): Promise { + const tenancy = await getTenancy(batch.tenancyId) ?? throwErr("Tenancy not found in processTenancyBatch? Was the tenancy deletion not cascaded?"); + + const prisma = await getPrismaClientForTenancy(tenancy); + const emailConfig = await getEmailConfig(tenancy); + + const userIds = new Set(); + for (const row of batch.rows) { + const recipient = deserializeRecipient(row.to as Json); + if ("userId" in recipient) { + userIds.add(recipient.userId); + } + } + + const users = userIds.size > 0 ? await prisma.projectUser.findMany({ + where: { + tenancyId: tenancy.id, + projectUserId: { in: Array.from(userIds) }, + }, + include: { + contactChannels: true, + }, + }) : []; + + const userMap = new Map(users.map(user => [user.projectUserId, user])); + + const context: TenancyProcessingContext = { + tenancy, + prisma, + emailConfig, + userMap, + notificationPreferences: new Map(), + }; + + const promises = batch.rows.map((row) => processSingleEmail(context, row)); + await allPromisesAndWaitUntilEach(promises); +} + +function getPrimaryEmail(user: ProjectUserWithContacts | undefined): string | undefined { + if (!user) return undefined; + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + const primaryChannel = user.contactChannels.find((channel) => channel.type === "EMAIL" && channel.isPrimary === "TRUE"); + return primaryChannel?.value ?? undefined; +} + +type ResolvedRecipient = + | { status: "ok", emails: string[] } + | { status: "skip", reason: EmailOutboxSkippedReason } + | { status: "unsubscribe" }; + +async function processSingleEmail(context: TenancyProcessingContext, row: EmailOutbox): Promise { + try { + const recipient = deserializeRecipient(row.to as Json); + const resolution = await resolveRecipientEmails(context, row, recipient); + + if (resolution.status === "skip") { + await markSkipped(row, resolution.reason); + return; + } + + if (resolution.status === "unsubscribe") { + await markSkipped(row, EmailOutboxSkippedReason.USER_UNSUBSCRIBED); + return; + } + + const result = await lowLevelSendEmailDirectViaProvider({ + tenancyId: context.tenancy.id, + emailConfig: context.emailConfig, + to: resolution.emails, + subject: row.renderedSubject ?? "", + html: row.renderedHtml ?? undefined, + text: row.renderedText ?? undefined, + }); + + if (result.status === "error") { + await globalPrismaClient.emailOutbox.update({ + where: { + tenancyId_id: { + tenancyId: row.tenancyId, + id: row.id, + }, + }, + data: { + finishedSendingAt: new Date(), + canHaveDeliveryInfo: false, + sendServerErrorExternalMessage: result.error.message, + sendServerErrorExternalDetails: { errorType: result.error.errorType }, + sendServerErrorInternalMessage: result.error.message, + sendServerErrorInternalDetails: { rawError: errorToNiceString(result.error.rawError), errorType: result.error.errorType }, + }, + }); + } else { + await globalPrismaClient.emailOutbox.update({ + where: { + tenancyId_id: { + tenancyId: row.tenancyId, + id: row.id, + }, + finishedSendingAt: null, + }, + data: { + finishedSendingAt: new Date(), + canHaveDeliveryInfo: false, + sendServerErrorExternalMessage: null, + sendServerErrorExternalDetails: Prisma.DbNull, + sendServerErrorInternalMessage: null, + sendServerErrorInternalDetails: Prisma.DbNull, + }, + }); + } + } catch (error) { + captureError("email-queue-step-sending-single-error", error); + await globalPrismaClient.emailOutbox.update({ + where: { + tenancyId_id: { + tenancyId: row.tenancyId, + id: row.id, + }, + finishedSendingAt: null, + }, + data: { + finishedSendingAt: new Date(), + sendServerErrorExternalMessage: "An error occurred while sending the email. If you are the admin of this project, please check the email configuration and try again.", + sendServerErrorExternalDetails: {}, + sendServerErrorInternalMessage: errorToNiceString(error), + sendServerErrorInternalDetails: {}, + }, + }); + } +} + +async function resolveRecipientEmails( + context: TenancyProcessingContext, + row: EmailOutbox, + recipient: ReturnType, +): Promise { + if (recipient.type === "custom-emails") { + if (recipient.emails.length === 0) { + return { status: "skip", reason: EmailOutboxSkippedReason.USER_DELETED_ACCOUNT }; + } + return { status: "ok", emails: recipient.emails }; + } + + const user = context.userMap.get(recipient.userId); + if (!user) { + return { status: "skip", reason: EmailOutboxSkippedReason.USER_DELETED_ACCOUNT }; + } + + const primaryEmail = getPrimaryEmail(user); + let emails: string[] = []; + if (recipient.type === "user-custom-emails") { + emails = recipient.emails.length > 0 ? recipient.emails : primaryEmail ? [primaryEmail] : []; + } else { + emails = primaryEmail ? [primaryEmail] : []; + } + + if (emails.length === 0) { + return { status: "skip", reason: EmailOutboxSkippedReason.USER_DELETED_ACCOUNT }; + } + + if (row.renderedNotificationCategoryId) { + const canSend = await shouldSendEmail(context, row.renderedNotificationCategoryId, recipient.userId); + if (!canSend) { + return { status: "unsubscribe" }; + } + } + + return { status: "ok", emails }; +} + +async function shouldSendEmail( + context: TenancyProcessingContext, + categoryId: string, + userId: string, +): Promise { + const category = getNotificationCategoryById(categoryId); + if (!category) { + return true; + } + if (!category.can_disable) { + return true; + } + + let cache = context.notificationPreferences.get(categoryId); + if (!cache) { + const userIds = Array.from(context.userMap.keys()); + if (userIds.length === 0) { + cache = new Map(); + } else { + const preferences = await context.prisma.userNotificationPreference.findMany({ + where: { + tenancyId: context.tenancy.id, + notificationCategoryId: categoryId, + projectUserId: { in: userIds }, + }, + }); + cache = new Map(preferences.map(pref => [pref.projectUserId, pref.enabled])); + } + context.notificationPreferences.set(categoryId, cache); + } + + if (!cache.has(userId)) { + return category.default_enabled; + } + return cache.get(userId) ?? category.default_enabled; +} + +async function markSkipped(row: EmailOutbox, reason: EmailOutboxSkippedReason): Promise { + await globalPrismaClient.emailOutbox.update({ + where: { + tenancyId_id: { + tenancyId: row.tenancyId, + id: row.id, + }, + }, + data: { + skippedReason: reason, + finishedSendingAt: new Date(), + canHaveDeliveryInfo: false, + }, + }); +} + + +export function serializeRecipient(recipient: EmailOutboxRecipient): Json { + switch (recipient.type) { + case "user-primary-email": { + return { + type: recipient.type, + userId: recipient.userId, + }; + } + case "user-custom-emails": { + return { + type: recipient.type, + userId: recipient.userId, + emails: recipient.emails, + }; + } + case "custom-emails": { + return { + type: recipient.type, + emails: recipient.emails, + }; + } + default: { + throw new StackAssertionError("Unknown EmailOutbox recipient type", { recipient }); + } + } +} + +export function deserializeRecipient(raw: Json): EmailOutboxRecipient { + if (raw === null || typeof raw !== "object" || Array.isArray(raw)) { + throw new StackAssertionError("Malformed EmailOutbox recipient payload", { raw }); + } + const base = raw as Record; + const type = base.type; + if (type === "user-primary-email") { + const userId = base.userId; + if (typeof userId !== "string") { + throw new StackAssertionError("Expected userId to be present for user-primary-email recipient", { raw }); + } + return { type, userId }; + } + if (type === "user-custom-emails") { + const userId = base.userId; + const emails = base.emails; + if (typeof userId !== "string" || !Array.isArray(emails) || !emails.every((item) => typeof item === "string")) { + throw new StackAssertionError("Invalid user-custom-emails recipient payload", { raw }); + } + return { type, userId, emails: emails as string[] }; + } + if (type === "custom-emails") { + const emails = base.emails; + if (!Array.isArray(emails) || !emails.every((item) => typeof item === "string")) { + throw new StackAssertionError("Invalid custom-emails recipient payload", { raw }); + } + return { type, emails: emails as string[] }; + } + throw new StackAssertionError("Unknown EmailOutbox recipient type", { raw }); +} diff --git a/apps/backend/src/lib/email-queue.tsx b/apps/backend/src/lib/email-queue.tsx new file mode 100644 index 0000000000..bf5568f68b --- /dev/null +++ b/apps/backend/src/lib/email-queue.tsx @@ -0,0 +1,27 @@ +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; +import { upstash } from "./upstash"; + +const EMAIL_QUEUE_FLOW_KEY = "stack-auth-email-queue-step-flow-key"; + +/** + * Enqueues the email queue step on QStash. The step is idempotent; if the publish fails we log and continue. + */ +export async function enqueueEmailQueueStep(): Promise { + const baseUrl = getEnvVariable("NEXT_PUBLIC_STACK_API_URL"); + const url = new URL("/api/v1/internal/email-queue-step", baseUrl); + try { + await upstash.publishJSON({ + url: url.toString(), + method: "POST", + body: {}, + flowControl: { + key: EMAIL_QUEUE_FLOW_KEY, + parallelism: 1, + rate: 1, + }, + }); + } catch (error) { + captureError("enqueue-email-queue-step", error); + } +} diff --git a/apps/backend/src/lib/email-rendering.test.tsx b/apps/backend/src/lib/email-rendering.test.tsx new file mode 100644 index 0000000000..66d44bed78 --- /dev/null +++ b/apps/backend/src/lib/email-rendering.test.tsx @@ -0,0 +1,466 @@ +import { describe, expect, it } from 'vitest'; +import { renderEmailsForTenancyBatched, type RenderEmailRequestForTenancy } from './email-rendering'; + +describe('renderEmailsForTenancyBatched', () => { + const createSimpleTemplateSource = (content: string) => ` + export const variablesSchema = (v: any) => v; + export function EmailTemplate({ variables, user, project }: any) { + return ( + <> +
${content}
+
{user.displayName}
+
{project.displayName}
+ {variables &&
{JSON.stringify(variables)}
} + + ); + } + `; + + const createTemplateWithSubject = (subject: string, content: string) => ` + import { Subject } from "@stackframe/emails"; + export const variablesSchema = (v: any) => v; + export function EmailTemplate({ variables, user, project }: any) { + return ( + <> + +
${content}
+
{user.displayName}
+ + ); + } + `; + + const createTemplateWithNotificationCategory = (category: string, content: string) => ` + import { NotificationCategory } from "@stackframe/emails"; + export const variablesSchema = (v: any) => v; + export function EmailTemplate({ variables, user, project }: any) { + return ( + <> + +
${content}
+ + ); + } + `; + + const createSimpleThemeSource = () => ` + export function EmailTheme({ children, unsubscribeLink }: any) { + return ( +
+
Email Header
+
{children}
+ {unsubscribeLink && } +
+ ); + } + `; + + const createMockRequest = ( + index: number, + overrides?: Partial + ): RenderEmailRequestForTenancy => ({ + templateSource: overrides?.templateSource ?? createSimpleTemplateSource(`Template content ${index}`), + themeSource: overrides?.themeSource ?? createSimpleThemeSource(), + input: { + user: { displayName: overrides?.input?.user.displayName ?? `User ${index}` }, + project: { displayName: overrides?.input?.project.displayName ?? `Project ${index}` }, + variables: overrides?.input ? overrides.input.variables : undefined, + unsubscribeLink: overrides?.input ? overrides.input.unsubscribeLink : `https://example.com/unsubscribe/${index}`, + }, + }); + + describe('empty array input', () => { + it('should return empty array for empty requests', async () => { + const result = await renderEmailsForTenancyBatched([]); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data).toEqual([]); + expect(result.data).toHaveLength(0); + } + }); + }); + + describe('single request', () => { + it('should successfully render email for single request', async () => { + const request = createMockRequest(1); + const result = await renderEmailsForTenancyBatched([request]); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data).toHaveLength(1); + expect(result.data[0].html).toBeDefined(); + expect(result.data[0].text).toBeDefined(); + expect(result.data[0].html).toContain('Template content 1'); + expect(result.data[0].html).toContain('User 1'); + expect(result.data[0].html).toContain('Project 1'); + expect(result.data[0].html).toContain('Email Header'); + expect(result.data[0].html).toContain('Unsubscribe'); + expect(result.data[0].text).toContain('User 1'); + } + }); + + it('should render email with subject when specified', async () => { + const request = createMockRequest(1, { + templateSource: createTemplateWithSubject('Test Subject', 'Email body content'), + }); + const result = await renderEmailsForTenancyBatched([request]); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data).toHaveLength(1); + expect(result.data[0].subject).toBe('Test Subject'); + expect(result.data[0].html).toContain('Email body content'); + } + }); + + it('should render email with notification category when specified', async () => { + const request = createMockRequest(1, { + templateSource: createTemplateWithNotificationCategory('Transactional', 'Transaction email'), + }); + const result = await renderEmailsForTenancyBatched([request]); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data).toHaveLength(1); + expect(result.data[0].notificationCategory).toBe('Transactional'); + expect(result.data[0].html).toContain('Transaction email'); + } + }); + + it('should handle request without variables', async () => { + const request = createMockRequest(1, { + input: { + user: { displayName: 'John Doe' }, + project: { displayName: 'My Project' }, + }, + }); + const result = await renderEmailsForTenancyBatched([request]); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data).toHaveLength(1); + expect(result.data[0].html).toContain('John Doe'); + expect(result.data[0].html).toContain('My Project'); + } + }); + + it('should handle request with variables', async () => { + const request = createMockRequest(1, { + input: { + user: { displayName: 'Jane Doe' }, + project: { displayName: 'Test Project' }, + variables: { greeting: 'Hello', name: 'World' }, + }, + }); + const result = await renderEmailsForTenancyBatched([request]); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data).toHaveLength(1); + expect(result.data[0].html).toContain('Jane Doe'); + expect(result.data[0].html).toContain('Test Project'); + } + }); + + it('should handle request without unsubscribe link', async () => { + const request = createMockRequest(1, { + input: { + user: { displayName: 'User 1' }, + project: { displayName: 'Project 1' }, + }, + }); + const result = await renderEmailsForTenancyBatched([request]); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data).toHaveLength(1); + expect(result.data[0].html).toBeDefined(); + } + }); + + it('should handle user with null displayName', async () => { + const request = createMockRequest(1, { + input: { + user: { displayName: null }, + project: { displayName: 'Project 1' }, + }, + }); + const result = await renderEmailsForTenancyBatched([request]); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data).toHaveLength(1); + expect(result.data[0].html).toBeDefined(); + } + }); + }); + + describe('multiple requests', () => { + it('should successfully render emails for multiple requests', async () => { + const requests = [ + createMockRequest(1), + createMockRequest(2), + createMockRequest(3), + ]; + const result = await renderEmailsForTenancyBatched(requests); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data).toHaveLength(3); + + expect(result.data[0].html).toContain('Template content 1'); + expect(result.data[0].html).toContain('User 1'); + expect(result.data[0].html).toContain('Project 1'); + + expect(result.data[1].html).toContain('Template content 2'); + expect(result.data[1].html).toContain('User 2'); + expect(result.data[1].html).toContain('Project 2'); + + expect(result.data[2].html).toContain('Template content 3'); + expect(result.data[2].html).toContain('User 3'); + expect(result.data[2].html).toContain('Project 3'); + } + }); + + it('should handle requests with different templates and themes', async () => { + const requests = [ + createMockRequest(1, { + templateSource: createSimpleTemplateSource('Custom Template 1'), + themeSource: ` + export function EmailTheme({ children }: any) { + return
{children}
; + } + `, + }), + createMockRequest(2, { + templateSource: createSimpleTemplateSource('Custom Template 2'), + themeSource: ` + export function EmailTheme({ children }: any) { + return
{children}
; + } + `, + }), + ]; + const result = await renderEmailsForTenancyBatched(requests); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data).toHaveLength(2); + expect(result.data[0].html).toContain('Custom Template 1'); + expect(result.data[0].html).toContain('custom-theme-1'); + expect(result.data[1].html).toContain('Custom Template 2'); + expect(result.data[1].html).toContain('custom-theme-2'); + } + }); + + it('should handle mixed requests with and without subjects', async () => { + const requests = [ + createMockRequest(1, { + templateSource: createTemplateWithSubject('Subject 1', 'Content 1'), + }), + createMockRequest(2, { + templateSource: createSimpleTemplateSource('Content 2'), + }), + createMockRequest(3, { + templateSource: createTemplateWithSubject('Subject 3', 'Content 3'), + }), + ]; + const result = await renderEmailsForTenancyBatched(requests); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data).toHaveLength(3); + expect(result.data[0].subject).toBe('Subject 1'); + expect(result.data[1].subject).toBeUndefined(); + expect(result.data[2].subject).toBe('Subject 3'); + } + }); + + it('should handle requests with different users and projects', async () => { + const requests = [ + createMockRequest(1, { + input: { + user: { displayName: 'Alice' }, + project: { displayName: 'Project A' }, + }, + }), + createMockRequest(2, { + input: { + user: { displayName: null }, + project: { displayName: 'Project B' }, + }, + }), + createMockRequest(3, { + input: { + user: { displayName: 'Charlie' }, + project: { displayName: 'Project C' }, + }, + }), + ]; + const result = await renderEmailsForTenancyBatched(requests); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data).toHaveLength(3); + expect(result.data[0].html).toContain('Alice'); + expect(result.data[0].html).toContain('Project A'); + expect(result.data[1].html).toContain('Project B'); + expect(result.data[2].html).toContain('Charlie'); + expect(result.data[2].html).toContain('Project C'); + } + }); + }); + + describe('error handling', () => { + it('should return error for invalid template syntax', async () => { + const request = createMockRequest(1, { + templateSource: 'invalid syntax {{{ not jsx', + }); + const result = await renderEmailsForTenancyBatched([request]); + + expect(result.status).toBe('error'); + if (result.status === 'error') { + expect(result.error).toBeDefined(); + expect(typeof result.error).toBe('string'); + } + }); + + it('should return error for invalid theme syntax', async () => { + const request = createMockRequest(1, { + themeSource: 'export function EmailTheme( { unclosed bracket', + }); + const result = await renderEmailsForTenancyBatched([request]); + + expect(result.status).toBe('error'); + if (result.status === 'error') { + expect(result.error).toBeDefined(); + } + }); + + it('should return error when template does not export EmailTemplate', async () => { + const request = createMockRequest(1, { + templateSource: ` + export const variablesSchema = (v: any) => v; + export function WrongName() { + return
Wrong function name
; + } + `, + }); + const result = await renderEmailsForTenancyBatched([request]); + + expect(result.status).toBe('error'); + if (result.status === 'error') { + expect(result.error).toBeDefined(); + } + }); + + it('should return error when theme does not export EmailTheme', async () => { + const request = createMockRequest(1, { + themeSource: ` + export function WrongThemeName({ children }: any) { + return
{children}
; + } + `, + }); + const result = await renderEmailsForTenancyBatched([request]); + + expect(result.status).toBe('error'); + if (result.status === 'error') { + expect(result.error).toBeDefined(); + } + }); + }); + + describe('text rendering', () => { + it('should render plain text version of email', async () => { + const request = createMockRequest(1, { + templateSource: createSimpleTemplateSource('Plain text content'), + }); + const result = await renderEmailsForTenancyBatched([request]); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data[0].text).toBeDefined(); + expect(result.data[0].text).toContain('Plain text content'); + expect(result.data[0].text).toContain('User 1'); + } + }); + + it('should render text for multiple emails', async () => { + const requests = [ + createMockRequest(1), + createMockRequest(2), + ]; + const result = await renderEmailsForTenancyBatched(requests); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data[0].text).toBeDefined(); + expect(result.data[1].text).toBeDefined(); + expect(result.data[0].text).not.toBe(result.data[1].text); + } + }); + }); + + describe('unsubscribe link handling', () => { + it('should include unsubscribe link when provided', async () => { + const request = createMockRequest(1, { + input: { + user: { displayName: 'User 1' }, + project: { displayName: 'Project 1' }, + unsubscribeLink: 'https://example.com/unsubscribe/abc123', + }, + }); + const result = await renderEmailsForTenancyBatched([request]); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data[0].html).toContain('https://example.com/unsubscribe/abc123'); + } + }); + + it('should handle missing unsubscribe link gracefully', async () => { + const customTheme = ` + export function EmailTheme({ children, unsubscribeLink }: any) { + return ( +
+
{children}
+ {unsubscribeLink ? : null} +
+ ); + } + `; + const request = createMockRequest(1, { + themeSource: customTheme, + input: { + user: { displayName: 'User 1' }, + project: { displayName: 'Project 1' }, + }, + }); + const result = await renderEmailsForTenancyBatched([request]); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data[0].html).toBeDefined(); + } + }); + }); + + describe('large batch', () => { + it('should handle rendering 10 emails in a single batch', async () => { + const requests = Array.from({ length: 10 }, (_, i) => createMockRequest(i + 1)); + const result = await renderEmailsForTenancyBatched(requests); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data).toHaveLength(10); + result.data.forEach((email, i) => { + expect(email.html).toContain(`User ${i + 1}`); + expect(email.html).toContain(`Project ${i + 1}`); + expect(email.text).toBeDefined(); + }); + } + }, 30000); // Extended timeout for large batch + }); +}); diff --git a/apps/backend/src/lib/email-rendering.tsx b/apps/backend/src/lib/email-rendering.tsx index f1116eb54c..315566f201 100644 --- a/apps/backend/src/lib/email-rendering.tsx +++ b/apps/backend/src/lib/email-rendering.tsx @@ -1,12 +1,12 @@ import { Freestyle } from '@/lib/freestyle'; import { emptyEmailTheme } from '@stackframe/stack-shared/dist/helpers/emails'; +import { getEnvVariable } from '@stackframe/stack-shared/dist/utils/env'; import { captureError, StackAssertionError } from '@stackframe/stack-shared/dist/utils/errors'; import { bundleJavaScript } from '@stackframe/stack-shared/dist/utils/esbuild'; import { get, has } from '@stackframe/stack-shared/dist/utils/objects'; import { Result } from "@stackframe/stack-shared/dist/utils/results"; import { deindent } from "@stackframe/stack-shared/dist/utils/strings"; import { Tenancy } from './tenancies'; -import { getEnvVariable } from '@stackframe/stack-shared/dist/utils/env'; export function getActiveEmailTheme(tenancy: Tenancy) { const themeList = tenancy.config.emails.themes; @@ -20,12 +20,17 @@ export function getActiveEmailTheme(tenancy: Tenancy) { return get(themeList, currentActiveTheme); } -export function getEmailThemeForTemplate(tenancy: Tenancy, templateThemeId: string | null | false | undefined) { +/** + * If themeId is a string, and it is a valid theme id, return the theme's tsxSource. + * If themeId is false, return the empty email theme. + * If themeId is null or undefined, return the currently active email theme. + */ +export function getEmailThemeForThemeId(tenancy: Tenancy, themeId: string | null | false | undefined) { const themeList = tenancy.config.emails.themes; - if (templateThemeId && has(themeList, templateThemeId)) { - return get(themeList, templateThemeId).tsxSource; + if (themeId && has(themeList, themeId)) { + return get(themeList, themeId).tsxSource; } - if (templateThemeId === false) { + if (themeId === false) { return emptyEmailTheme; } return getActiveEmailTheme(tenancy).tsxSource; @@ -228,6 +233,118 @@ export async function renderEmailsWithTemplateBatched( return Result.ok(executeResult.data.result as Array<{ html: string, text: string, subject?: string, notificationCategory?: string }>); } +export type RenderEmailRequestForTenancy = { + templateSource: string, + themeSource: string, + input: { + user: { displayName: string | null }, + project: { displayName: string }, + variables?: Record, + unsubscribeLink?: string, + }, +}; + +export async function renderEmailsForTenancyBatched(requests: RenderEmailRequestForTenancy[]): Promise, string>> { + if (requests.length === 0) { + return Result.ok([]); + } + + const apiKey = getEnvVariable("STACK_FREESTYLE_API_KEY"); + const files: Record = { + "/utils.tsx": findComponentValueUtil, + }; + + for (let index = 0; index < requests.length; index++) { + const request = requests[index]; + files[`/template-${index}.tsx`] = request.templateSource; + files[`/theme-${index}.tsx`] = request.themeSource; + } + + const serializedInputs = JSON.stringify(requests.map((request) => ({ + user: request.input.user, + project: request.input.project, + variables: request.input.variables ?? null, + unsubscribeLink: request.input.unsubscribeLink ?? null, + }))); + + files["/render.tsx"] = deindent` + import { configure } from "arktype/config"; + configure({ onUndeclaredKey: "delete" }); + import React from "react"; + import { render } from "@react-email/components"; + import { type } from "arktype"; + import { findComponentValue } from "./utils.tsx"; + ${requests.map((_, index) => `import * as TemplateModule${index} from "./template-${index}.tsx";`).join("\n")} + ${requests.map((_, index) => `const { variablesSchema: variablesSchema${index}, EmailTemplate: EmailTemplate${index} } = TemplateModule${index};`).join("\n")} + ${requests.map((_, index) => `import { EmailTheme as EmailTheme${index} } from "./theme-${index}.tsx";`).join("\n")} + + export const renderAll = async () => { + const inputs = ${serializedInputs}; + const results = []; + ${requests.map((_, index) => deindent` + { + const input = inputs[${index}]; + const schema = variablesSchema${index}; + const variables = schema ? schema({ ...(input.variables || {}) }) : {}; + if (variables instanceof type.errors) { + throw new Error(variables.summary); + } + const TemplateWithProps = ; + const Email = + {TemplateWithProps} + ; + results.push({ + html: await render(Email), + text: await render(Email, { plainText: true }), + subject: findComponentValue(TemplateWithProps, "Subject"), + notificationCategory: findComponentValue(TemplateWithProps, "NotificationCategory"), + }); + } + `).join("\n")} + return results; + }; + `; + + files["/entry.js"] = deindent` + import { renderAll } from "./render.tsx"; + export default renderAll; + `; + + const bundle = await bundleJavaScript(files as Record & { '/entry.js': string }, { + keepAsImports: ["arktype", "react", "react/jsx-runtime", "@react-email/components"], + externalPackages: { "@stackframe/emails": stackframeEmailsPackage }, + format: "esm", + sourcemap: false, + }); + + if (bundle.status === "error") { + return Result.error(bundle.error); + } + + const freestyle = new Freestyle({ apiKey }); + const nodeModules = { + "react-dom": "19.1.1", + "react": "19.1.1", + "@react-email/components": "0.1.1", + "arktype": "2.1.20", + }; + + const execution = await freestyle.executeScript(bundle.data, { nodeModules }); + if (execution.status === "error") { + return Result.error(execution.error); + } + if (!execution.data.result) { + const noResultError = new StackAssertionError("No result from Freestyle", { + execution, + requests, + }); + captureError("freestyle-no-result", noResultError); + throw noResultError; + } + + return Result.ok(execution.data.result as Array<{ html: string, text: string, subject?: string, notificationCategory?: string }>); +} + const findComponentValueUtil = `import React from 'react'; export function findComponentValue(element, targetStackComponent) { const matches = []; diff --git a/apps/backend/src/lib/emails-low-level.tsx b/apps/backend/src/lib/emails-low-level.tsx new file mode 100644 index 0000000000..4f2b5f9731 --- /dev/null +++ b/apps/backend/src/lib/emails-low-level.tsx @@ -0,0 +1,350 @@ +/** + * + * Low-level email sending functions that bypass the email outbox queue and send directly via SMTP or email service + * providers. You probably shouldn't use this and should instead use the functions in emails.tsx. + */ + +import { getEnvVariable } from '@stackframe/stack-shared/dist/utils/env'; +import { StackAssertionError, captureError } from '@stackframe/stack-shared/dist/utils/errors'; +import { omit, pick } from '@stackframe/stack-shared/dist/utils/objects'; +import { runAsynchronously, wait } from '@stackframe/stack-shared/dist/utils/promises'; +import { Result } from '@stackframe/stack-shared/dist/utils/results'; +import { traceSpan } from '@stackframe/stack-shared/dist/utils/telemetry'; +import nodemailer from 'nodemailer'; +import { Resend } from 'resend'; +import { getTenancy } from './tenancies'; + +export function isSecureEmailPort(port: number | string) { + let parsedPort = parseInt(port.toString()); + return parsedPort === 465; +} + +export type LowLevelEmailConfig = { + host: string, + port: number, + username: string, + password: string, + senderEmail: string, + senderName: string, + secure: boolean, + type: 'shared' | 'standard', +} + +export type LowLevelSendEmailOptions = { + tenancyId: string, + emailConfig: LowLevelEmailConfig, + to: string | string[], + subject: string, + html?: string, + text?: string, +} + +async function _lowLevelSendEmailWithoutRetries(options: LowLevelSendEmailOptions): Promise> { + let finished = false; + runAsynchronously(async () => { + await wait(10000); + if (!finished) { + captureError("email-send-timeout", new StackAssertionError("Email send took longer than 10s; maybe the email service is too slow?", { + config: options.emailConfig.type === 'shared' ? "shared" : pick(options.emailConfig, ['host', 'port', 'username', 'senderEmail', 'senderName']), + to: options.to, + subject: options.subject, + html: options.html, + text: options.text, + })); + } + }); + try { + let toArray = typeof options.to === 'string' ? [options.to] : options.to; + + // If using the shared email config, use Emailable to check if the email is valid. skip the ones that are not (it's as if they had bounced) + const emailableApiKey = getEnvVariable('STACK_EMAILABLE_API_KEY', ""); + if (options.emailConfig.type === 'shared' && emailableApiKey) { + await traceSpan('verifying email addresses with Emailable', async () => { + toArray = (await Promise.all(toArray.map(async (to) => { + try { + const emailableResponseResult = await Result.retry(async (attempt) => { + const res = await fetch(`https://api.emailable.com/v1/verify?email=${encodeURIComponent(options.to as string)}&api_key=${emailableApiKey}`); + if (res.status === 249) { + const text = await res.text(); + console.log('Emailable is taking longer than expected, retrying...', text, { to: options.to }); + return Result.error(new Error("Emailable API returned a 249 error for " + options.to + ". This means it takes some more time to verify the email address. Response body: " + text)); + } + return Result.ok(res); + }, 4, { exponentialDelayBase: 4000 }); + if (emailableResponseResult.status === 'error') { + throw new StackAssertionError("Timed out while verifying email address with Emailable", { + to: options.to, + emailableResponseResult, + }); + } + const emailableResponse = emailableResponseResult.data; + if (!emailableResponse.ok) { + throw new StackAssertionError("Failed to verify email address with Emailable", { + to: options.to, + emailableResponse, + emailableResponseText: await emailableResponse.text(), + }); + } + const json = await emailableResponse.json(); + console.log('emailableResponse', json); + if (json.state === 'undeliverable' || json.disposable) { + console.log('email not deliverable', to, json); + return null; + } + return to; + } catch (error) { + // if something goes wrong with the Emailable API (eg. 500, ran out of credits, etc.), we just send the email anyway + captureError("emailable-api-error", error); + return to; + } + }))).filter((to): to is string => to !== null); + }); + } + + if (toArray.length === 0) { + // no valid emails, so we can just return ok + // (we skip silently because this is not an error) + return Result.ok(undefined); + } + + return await traceSpan('sending email to ' + JSON.stringify(toArray), async () => { + try { + const transporter = nodemailer.createTransport({ + host: options.emailConfig.host, + port: options.emailConfig.port, + secure: options.emailConfig.secure, + auth: { + user: options.emailConfig.username, + pass: options.emailConfig.password, + }, + }); + + await transporter.sendMail({ + from: `"${options.emailConfig.senderName}" <${options.emailConfig.senderEmail}>`, + ...options, + to: toArray, + }); + + return Result.ok(undefined); + } catch (error) { + if (error instanceof Error) { + const code = (error as any).code as string | undefined; + const responseCode = (error as any).responseCode as number | undefined; + const errorNumber = (error as any).errno as number | undefined; + + const getServerResponse = (error: any) => { + if (error?.response) { + return `\nResponse from the email server:\n${error.response}`; + } + return ''; + }; + + if (errorNumber === -3008 || code === 'EDNS') { + return Result.error({ + rawError: error, + errorType: 'HOST_NOT_FOUND', + canRetry: false, + message: 'Failed to connect to the email host. Please make sure the email host configuration is correct.' + } as const); + } + + if (responseCode === 535 || code === 'EAUTH') { + return Result.error({ + rawError: error, + errorType: 'AUTH_FAILED', + canRetry: false, + message: 'Failed to authenticate with the email server. Please check your email credentials configuration.', + } as const); + } + + if (responseCode === 450) { + return Result.error({ + rawError: error, + errorType: 'TEMPORARY', + canRetry: true, + message: 'The email server returned a temporary error. This could be due to a temporary network issue or a temporary block on the email server. Please try again later.\n\nError: ' + getServerResponse(error), + } as const); + } + + if (responseCode === 553) { + return Result.error({ + rawError: error, + errorType: 'INVALID_EMAIL_ADDRESS', + canRetry: false, + message: 'The email address provided is invalid. Please verify both the recipient and sender email addresses configuration are correct.\n\nError:' + getServerResponse(error), + } as const); + } + + if (responseCode === 554 || code === 'EENVELOPE') { + return Result.error({ + rawError: error, + errorType: 'REJECTED', + canRetry: false, + message: 'The email server rejected the email. Please check your email configuration and try again later.\n\nError:' + getServerResponse(error), + } as const); + } + + if (code === 'ETIMEDOUT') { + return Result.error({ + rawError: error, + errorType: 'TIMEOUT', + canRetry: true, + message: 'The email server timed out while sending the email. This could be due to a temporary network issue or a temporary block on the email server. Please try again later.', + } as const); + } + + if (error.message.includes('Unexpected socket close')) { + return Result.error({ + rawError: error, + errorType: 'SOCKET_CLOSED', + canRetry: false, + message: 'Connection to email server was lost unexpectedly. This could be due to incorrect email server port configuration or a temporary network issue. Please verify your configuration and try again.', + } as const); + } + } + + // ============ temporary error ============ + const temporaryErrorIndicators = [ + "450 ", + "Client network socket disconnected before secure TLS connection was established", + "Too many requests", + ...options.emailConfig.host.includes("resend") ? [ + // Resend is a bit unreliable, so we'll retry even in some cases where it may send duplicate emails + "ECONNRESET", + ] : [], + ]; + if (temporaryErrorIndicators.some(indicator => error instanceof Error && error.message.includes(indicator))) { + // this can happen occasionally (especially with certain unreliable email providers) + // so let's retry + return Result.error({ + rawError: error, + errorType: 'UNKNOWN', + canRetry: true, + message: 'Failed to send email, but error is possibly transient due to the internet connection. Please check your email configuration and try again later.', + } as const); + } + + // ============ unknown error ============ + return Result.error({ + rawError: error, + errorType: 'UNKNOWN', + canRetry: false, + message: 'An unknown error occurred while sending the email.', + } as const); + } + }); + } finally { + finished = true; + } +} + +export async function lowLevelSendEmailDirectWithoutRetries(options: LowLevelSendEmailOptions): Promise> { + return await _lowLevelSendEmailWithoutRetries(options); +} + +export async function lowLevelSendEmailResendBatchedDirect(resendApiKey: string, emailOptions: LowLevelSendEmailOptions[]) { + if (emailOptions.length === 0) { + return Result.ok([]); + } + if (emailOptions.length > 100) { + throw new StackAssertionError("sendEmailResendBatchedDirect expects at most 100 emails to be sent at once", { emailOptions }); + } + if (emailOptions.some(option => option.tenancyId !== emailOptions[0].tenancyId)) { + throw new StackAssertionError("sendEmailResendBatchedDirect expects all emails to be sent from the same tenancy", { emailOptions }); + } + const tenancy = await getTenancy(emailOptions[0].tenancyId); + if (!tenancy) { + throw new StackAssertionError("Tenancy not found"); + } + const resend = new Resend(resendApiKey); + const result = await Result.retry(async (_) => { + const { data, error } = await resend.batch.send(emailOptions.map((option) => ({ + from: option.emailConfig.senderEmail, + to: option.to, + subject: option.subject, + html: option.html ?? "", + text: option.text, + }))); + + if (data) { + return Result.ok(data.data); + } + if (error.name === "rate_limit_exceeded" || error.name === "internal_server_error") { + return Result.error(error); + } + return Result.ok(null); + }, 3, { exponentialDelayBase: 2000 }); + + return result; +} + +export async function lowLevelSendEmailDirectViaProvider(options: LowLevelSendEmailOptions): Promise> { + if (!options.to) { + throw new StackAssertionError("No recipient email address provided to sendEmail", omit(options, ['emailConfig'])); + } + + const errorMessage = "Failed to send email. If you are the admin of this project, please check the email configuration and try again."; + + class DoNotRetryError extends Error { + constructor(public readonly errorObj: { + rawError: any, + errorType: string, + canRetry: boolean, + message?: string, + }) { + super("This error should never be caught anywhere else but inside the lowLevelSendEmailDirectViaProvider function, something went wrong if you see this!"); + } + } + + let result; + try { + result = await Result.retry(async (attempt) => { + const result = await lowLevelSendEmailDirectWithoutRetries(options); + + if (result.status === 'error') { + const extraData = { + host: options.emailConfig.host, + from: options.emailConfig.senderEmail, + to: options.to, + subject: options.subject, + error: result.error, + }; + + if (result.error.canRetry) { + console.warn("Failed to send email, but error is possibly transient so retrying.", extraData, result.error.rawError); + return Result.error(result.error); + } + + console.warn("Failed to send email, and error is not transient, so not retrying.", extraData, result.error.rawError); + throw new DoNotRetryError(result.error); + } + + return result; + }, 3, { exponentialDelayBase: 2000 }); + } catch (error) { + if (error instanceof DoNotRetryError) { + return Result.error(error.errorObj); + } + throw error; + } + + if (result.status === 'error') { + return Result.error(result.error.errors[0]); + } + return Result.ok(undefined); +} diff --git a/apps/backend/src/lib/emails.tsx b/apps/backend/src/lib/emails.tsx index 06f32253c5..6f1d10bd87 100644 --- a/apps/backend/src/lib/emails.tsx +++ b/apps/backend/src/lib/emails.tsx @@ -1,17 +1,27 @@ -import { getPrismaClientForTenancy } from '@/prisma-client'; +import { globalPrismaClient } from '@/prisma-client'; +import { runAsynchronouslyAndWaitUntil } from '@/utils/vercel'; +import { EmailOutboxCreatedWith } from '@prisma/client'; import { DEFAULT_TEMPLATE_IDS } from '@stackframe/stack-shared/dist/helpers/emails'; import { UsersCrud } from '@stackframe/stack-shared/dist/interface/crud/users'; import { getEnvVariable } from '@stackframe/stack-shared/dist/utils/env'; -import { StackAssertionError, StatusError, captureError } from '@stackframe/stack-shared/dist/utils/errors'; -import { filterUndefined, omit, pick } from '@stackframe/stack-shared/dist/utils/objects'; -import { runAsynchronously, wait } from '@stackframe/stack-shared/dist/utils/promises'; -import { Result } from '@stackframe/stack-shared/dist/utils/results'; -import { traceSpan } from '@stackframe/stack-shared/dist/utils/telemetry'; -import nodemailer from 'nodemailer'; -import { getEmailThemeForTemplate, renderEmailWithTemplate } from './email-rendering'; -import { Tenancy, getTenancy } from './tenancies'; -import { Resend } from 'resend'; - +import { StackAssertionError } from '@stackframe/stack-shared/dist/utils/errors'; +import { runEmailQueueStep } from './email-queue-step'; +import { getEmailThemeForThemeId } from './email-rendering'; +import { LowLevelEmailConfig, isSecureEmailPort } from './emails-low-level'; +import { Tenancy } from './tenancies'; + + +/** + * Describes where an email should be delivered. Each outbox entry targets exactly one recipient entity. + * + * user-primary-email: the email is being sent to the primary email address of a user (determined at the time of sending, NOT the time of creation/rendering). if the user unsubscribes, they will not receive the email. + * user-custom-emails: the email is being sent to a list of custom emails, but if the user unsubscribes, they will no longer receive the email. + * custom-emails: the email is being sent to a list of custom emails. there is no associated user object and the recipient cannot unsubscribe. cannot be used to send non-transactional emails. + */ +export type EmailOutboxRecipient = + | { type: "user-primary-email", userId: string } + | { type: "user-custom-emails", userId: string, emails: string[] } + | { type: "custom-emails", emails: string[] }; function getDefaultEmailTemplate(tenancy: Tenancy, type: keyof typeof DEFAULT_TEMPLATE_IDS) { const templateList = new Map(Object.entries(tenancy.config.emails.templates)); @@ -27,399 +37,63 @@ function getDefaultEmailTemplate(tenancy: Tenancy, type: keyof typeof DEFAULT_TE throw new StackAssertionError(`Unknown email template type: ${type}`); } -export function isSecureEmailPort(port: number | string) { - let parsedPort = parseInt(port.toString()); - return parsedPort === 465; -} - -export type EmailConfig = { - host: string, - port: number, - username: string, - password: string, - senderEmail: string, - senderName: string, - secure: boolean, - type: 'shared' | 'standard', -} - -type SendEmailOptions = { - tenancyId: string, - emailConfig: EmailConfig, - to: string | string[], - subject: string, - html?: string, - text?: string, -} - -async function _sendEmailWithoutRetries(options: SendEmailOptions): Promise> { - let finished = false; - runAsynchronously(async () => { - await wait(10000); - if (!finished) { - captureError("email-send-timeout", new StackAssertionError("Email send took longer than 10s; maybe the email service is too slow?", { - config: options.emailConfig.type === 'shared' ? "shared" : pick(options.emailConfig, ['host', 'port', 'username', 'senderEmail', 'senderName']), - to: options.to, - subject: options.subject, - html: options.html, - text: options.text, - })); - } - }); - try { - let toArray = typeof options.to === 'string' ? [options.to] : options.to; - - // If using the shared email config, use Emailable to check if the email is valid. skip the ones that are not (it's as if they had bounced) - const emailableApiKey = getEnvVariable('STACK_EMAILABLE_API_KEY', ""); - if (options.emailConfig.type === 'shared' && emailableApiKey) { - await traceSpan('verifying email addresses with Emailable', async () => { - toArray = (await Promise.all(toArray.map(async (to) => { - try { - const emailableResponseResult = await Result.retry(async (attempt) => { - const res = await fetch(`https://api.emailable.com/v1/verify?email=${encodeURIComponent(options.to as string)}&api_key=${emailableApiKey}`); - if (res.status === 249) { - const text = await res.text(); - console.log('Emailable is taking longer than expected, retrying...', text, { to: options.to }); - return Result.error(new Error("Emailable API returned a 249 error for " + options.to + ". This means it takes some more time to verify the email address. Response body: " + text)); - } - return Result.ok(res); - }, 4, { exponentialDelayBase: 4000 }); - if (emailableResponseResult.status === 'error') { - throw new StackAssertionError("Timed out while verifying email address with Emailable", { - to: options.to, - emailableResponseResult, - }); - } - const emailableResponse = emailableResponseResult.data; - if (!emailableResponse.ok) { - throw new StackAssertionError("Failed to verify email address with Emailable", { - to: options.to, - emailableResponse, - emailableResponseText: await emailableResponse.text(), - }); - } - const json = await emailableResponse.json(); - console.log('emailableResponse', json); - if (json.state === 'undeliverable' || json.disposable) { - console.log('email not deliverable', to, json); - return null; - } - return to; - } catch (error) { - // if something goes wrong with the Emailable API (eg. 500, ran out of credits, etc.), we just send the email anyway - captureError("emailable-api-error", error); - return to; - } - }))).filter((to): to is string => to !== null); - }); - } - - if (toArray.length === 0) { - // no valid emails, so we can just return ok - // (we skip silently because this is not an error) - return Result.ok(undefined); - } - - return await traceSpan('sending email to ' + JSON.stringify(toArray), async () => { - try { - const transporter = nodemailer.createTransport({ - host: options.emailConfig.host, - port: options.emailConfig.port, - secure: options.emailConfig.secure, - auth: { - user: options.emailConfig.username, - pass: options.emailConfig.password, - }, - }); - - await transporter.sendMail({ - from: `"${options.emailConfig.senderName}" <${options.emailConfig.senderEmail}>`, - ...options, - to: toArray, - }); - - return Result.ok(undefined); - } catch (error) { - if (error instanceof Error) { - const code = (error as any).code as string | undefined; - const responseCode = (error as any).responseCode as number | undefined; - const errorNumber = (error as any).errno as number | undefined; - - const getServerResponse = (error: any) => { - if (error?.response) { - return `\nResponse from the email server:\n${error.response}`; - } - return ''; - }; - - if (errorNumber === -3008 || code === 'EDNS') { - return Result.error({ - rawError: error, - errorType: 'HOST_NOT_FOUND', - canRetry: false, - message: 'Failed to connect to the email host. Please make sure the email host configuration is correct.' - } as const); - } - - if (responseCode === 535 || code === 'EAUTH') { - return Result.error({ - rawError: error, - errorType: 'AUTH_FAILED', - canRetry: false, - message: 'Failed to authenticate with the email server. Please check your email credentials configuration.', - } as const); - } - - if (responseCode === 450) { - return Result.error({ - rawError: error, - errorType: 'TEMPORARY', - canRetry: true, - message: 'The email server returned a temporary error. This could be due to a temporary network issue or a temporary block on the email server. Please try again later.\n\nError: ' + getServerResponse(error), - } as const); - } - - if (responseCode === 553) { - return Result.error({ - rawError: error, - errorType: 'INVALID_EMAIL_ADDRESS', - canRetry: false, - message: 'The email address provided is invalid. Please verify both the recipient and sender email addresses configuration are correct.\n\nError:' + getServerResponse(error), - } as const); - } - - if (responseCode === 554 || code === 'EENVELOPE') { - return Result.error({ - rawError: error, - errorType: 'REJECTED', - canRetry: false, - message: 'The email server rejected the email. Please check your email configuration and try again later.\n\nError:' + getServerResponse(error), - } as const); - } - - if (code === 'ETIMEDOUT') { - return Result.error({ - rawError: error, - errorType: 'TIMEOUT', - canRetry: true, - message: 'The email server timed out while sending the email. This could be due to a temporary network issue or a temporary block on the email server. Please try again later.', - } as const); - } - - if (error.message.includes('Unexpected socket close')) { - return Result.error({ - rawError: error, - errorType: 'SOCKET_CLOSED', - canRetry: false, - message: 'Connection to email server was lost unexpectedly. This could be due to incorrect email server port configuration or a temporary network issue. Please verify your configuration and try again.', - } as const); - } - } - - // ============ temporary error ============ - const temporaryErrorIndicators = [ - "450 ", - "Client network socket disconnected before secure TLS connection was established", - "Too many requests", - ...options.emailConfig.host.includes("resend") ? [ - // Resend is a bit unreliable, so we'll retry even in some cases where it may send duplicate emails - "ECONNRESET", - ] : [], - ]; - if (temporaryErrorIndicators.some(indicator => error instanceof Error && error.message.includes(indicator))) { - // this can happen occasionally (especially with certain unreliable email providers) - // so let's retry - return Result.error({ - rawError: error, - errorType: 'UNKNOWN', - canRetry: true, - message: 'Failed to send email, but error is possibly transient due to the internet connection. Please check your email configuration and try again later.', - } as const); - } - - // ============ unknown error ============ - return Result.error({ - rawError: error, - errorType: 'UNKNOWN', - canRetry: false, - message: 'An unknown error occurred while sending the email.', - } as const); - } - }); - } finally { - finished = true; - } -} - -export async function sendEmailWithoutRetries(options: SendEmailOptions): Promise> { - const res = await _sendEmailWithoutRetries(options); - const tenancy = await getTenancy(options.tenancyId); - if (!tenancy) { - throw new StackAssertionError("Tenancy not found"); - } - - const prisma = await getPrismaClientForTenancy(tenancy); - - await prisma.sentEmail.create({ - data: { - tenancyId: options.tenancyId, - to: typeof options.to === 'string' ? [options.to] : options.to, - subject: options.subject, - html: options.html, - text: options.text, - senderConfig: omit(options.emailConfig, ['password']), - error: res.status === 'error' ? res.error : undefined, - }, - }); - return res; -} - -export async function sendEmailResendBatched(resendApiKey: string, emailOptions: SendEmailOptions[]) { - if (emailOptions.length === 0) { - return Result.ok([]); - } - if (emailOptions.length > 100) { - throw new StackAssertionError("sendEmailResendBatched expects at most 100 emails to be sent at once", { emailOptions }); - } - if (emailOptions.some(option => option.tenancyId !== emailOptions[0].tenancyId)) { - throw new StackAssertionError("sendEmailResendBatched expects all emails to be sent from the same tenancy", { emailOptions }); - } - const tenancy = await getTenancy(emailOptions[0].tenancyId); - if (!tenancy) { - throw new StackAssertionError("Tenancy not found"); - } - const prisma = await getPrismaClientForTenancy(tenancy); - const resend = new Resend(resendApiKey); - const result = await Result.retry(async (_) => { - const { data, error } = await resend.batch.send(emailOptions.map((option) => ({ - from: option.emailConfig.senderEmail, - to: option.to, - subject: option.subject, - html: option.html ?? "", - text: option.text, - }))); - - if (data) { - return Result.ok(data.data); - } - if (error.name === "rate_limit_exceeded" || error.name === "internal_server_error") { - return Result.error(error); - } - return Result.ok(null); - }, 3, { exponentialDelayBase: 2000 }); - - await prisma.sentEmail.createMany({ - data: emailOptions.map((options) => ({ - tenancyId: options.tenancyId, - to: typeof options.to === 'string' ? [options.to] : options.to, - subject: options.subject, - html: options.html, - text: options.text, - senderConfig: omit(options.emailConfig, ['password']), - error: result.status === 'error' ? result.error.message : undefined, +export async function sendEmailToMany(options: { + tenancy: Tenancy, + recipients: EmailOutboxRecipient[], + tsxSource: string, + extraVariables: Record, + themeId: string | null, + isHighPriority: boolean, + shouldSkipDeliverabilityCheck: boolean, + scheduledAt: Date, + createdWith: { type: "draft", draftId: string } | { type: "programmatic-call", templateId: string | null }, +}) { + await globalPrismaClient.emailOutbox.createMany({ + data: options.recipients.map(recipient => ({ + tenancyId: options.tenancy.id, + tsxSource: options.tsxSource, + themeId: options.themeId, + isHighPriority: options.isHighPriority, + createdWith: options.createdWith.type === "draft" ? EmailOutboxCreatedWith.DRAFT : EmailOutboxCreatedWith.PROGRAMMATIC_CALL, + emailDraftId: options.createdWith.type === "draft" ? options.createdWith.draftId : undefined, + emailProgrammaticCallTemplateId: options.createdWith.type === "programmatic-call" ? options.createdWith.templateId : undefined, + to: recipient, + extraRenderVariables: options.extraVariables, + scheduledAt: options.scheduledAt, + shouldSkipDeliverabilityCheck: options.shouldSkipDeliverabilityCheck, })), }); - return result; + // The cron job should run runEmailQueueStep() to process the emails, but we call it here again for those self-hosters + // who didn't set up the cron job correctly, and also just in case something happens to the cron job. + runAsynchronouslyAndWaitUntil(runEmailQueueStep()); } -export async function sendEmail(options: SendEmailOptions) { - if (!options.to) { - throw new StackAssertionError("No recipient email address provided to sendEmail", omit(options, ['emailConfig'])); - } - - const errorMessage = "Failed to send email. If you are the admin of this project, please check the email configuration and try again."; - - const handleError = (error: any) => { - console.warn("Failed to send email", error); - if (options.emailConfig.type === 'shared') { - captureError("failed-to-send-email-to-shared-email-config", error); - } - throw new StatusError(400, errorMessage); - }; - - const result = await Result.retry(async (attempt) => { - const result = await sendEmailWithoutRetries(options); - - if (result.status === 'error') { - const extraData = { - host: options.emailConfig.host, - from: options.emailConfig.senderEmail, - to: options.to, - subject: options.subject, - error: result.error, - }; - - if (result.error.canRetry) { - console.warn("Failed to send email, but error is possibly transient so retrying.", extraData, result.error.rawError); - return Result.error(result.error); - } - - handleError(extraData); - } - - return result; - }, 3, { exponentialDelayBase: 2000 }); - - if (result.status === 'error') { - handleError(result.error); - } -} - -export async function sendEmailFromTemplate(options: { +export async function sendEmailFromDefaultTemplate(options: { tenancy: Tenancy, user: UsersCrud["Admin"]["Read"] | null, email: string, templateType: keyof typeof DEFAULT_TEMPLATE_IDS, extraVariables: Record, + shouldSkipDeliverabilityCheck: boolean, version?: 1 | 2, }) { const template = getDefaultEmailTemplate(options.tenancy, options.templateType); - const themeSource = getEmailThemeForTemplate(options.tenancy, template.themeId); - const variables = filterUndefined({ - projectDisplayName: options.tenancy.project.display_name, - userDisplayName: options.user?.display_name ?? "", - ...filterUndefined(options.extraVariables), - }); - - const result = await renderEmailWithTemplate( - template.tsxSource, - themeSource, - { - user: { displayName: options.user?.display_name ?? null }, - project: { displayName: options.tenancy.project.display_name }, - variables, - } - ); - if (result.status === 'error') { - throw new StackAssertionError("Failed to render email template", { - template: template, - theme: themeSource, - variables, - result - }); - } - - await sendEmail({ - tenancyId: options.tenancy.id, - emailConfig: await getEmailConfig(options.tenancy), - to: options.email, - subject: result.data.subject ?? "", - html: result.data.html, - text: result.data.text, + const themeSource = getEmailThemeForThemeId(options.tenancy, template.themeId); + + await sendEmailToMany({ + tenancy: options.tenancy, + recipients: [options.user ? { type: "user-custom-emails", userId: options.user.id, emails: [options.email] } : { type: "custom-emails", emails: [options.email] }], + tsxSource: template.tsxSource, + extraVariables: options.extraVariables, + themeId: template.themeId === false ? null : options.tenancy.config.emails.selectedThemeId, + createdWith: { type: "programmatic-call", templateId: DEFAULT_TEMPLATE_IDS[options.templateType] }, + isHighPriority: options.shouldSkipDeliverabilityCheck, // always make emails sent via default template high priority + shouldSkipDeliverabilityCheck: options.shouldSkipDeliverabilityCheck, + scheduledAt: new Date(), }); } -export async function getEmailConfig(tenancy: Tenancy): Promise { +export async function getEmailConfig(tenancy: Tenancy): Promise { const projectEmailConfig = tenancy.config.emails.server; if (projectEmailConfig.isShared) { @@ -442,7 +116,7 @@ export async function getEmailConfig(tenancy: Tenancy): Promise { } -export async function getSharedEmailConfig(displayName: string): Promise { +export async function getSharedEmailConfig(displayName: string): Promise { return { host: getEnvVariable('STACK_EMAIL_HOST'), port: parseInt(getEnvVariable('STACK_EMAIL_PORT')), diff --git a/apps/backend/src/lib/notification-categories.ts b/apps/backend/src/lib/notification-categories.ts index f39ae3a675..c79a8025d3 100644 --- a/apps/backend/src/lib/notification-categories.ts +++ b/apps/backend/src/lib/notification-categories.ts @@ -2,7 +2,7 @@ import { Tenancy } from "@/lib/tenancies"; import { getPrismaClientForTenancy } from "@/prisma-client"; import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; -import { signInVerificationCodeHandler } from "../app/api/latest/auth/otp/sign-in/verification-code-handler"; +import { unsubscribeLinkVerificationCodeHandler } from "../app/api/latest/emails/unsubscribe-link/verification-handler"; // For now, we only have two hardcoded notification categories. TODO: query from database instead and create UI to manage them in dashboard export const listNotificationCategories = () => { @@ -26,6 +26,10 @@ export const getNotificationCategoryByName = (name: string) => { return listNotificationCategories().find((category) => category.name === name); }; +export const getNotificationCategoryById = (id: string) => { + return listNotificationCategories().find((category) => category.id === id); +}; + export const hasNotificationEnabled = async (tenancy: Tenancy, userId: string, notificationCategoryId: string) => { const notificationCategory = listNotificationCategories().find((category) => category.id === notificationCategoryId); if (!notificationCategory) { @@ -48,13 +52,12 @@ export const hasNotificationEnabled = async (tenancy: Tenancy, userId: string, n }; export const generateUnsubscribeLink = async (tenancy: Tenancy, userId: string, notificationCategoryId: string) => { - const { code } = await signInVerificationCodeHandler.createCode({ + const { code } = await unsubscribeLinkVerificationCodeHandler.createCode({ tenancy, - expiresInMs: 1000 * 60 * 60 * 24 * 30, - data: {}, - method: { - email: "test@test.com", - type: "standard", + method: {}, + data: { + user_id: userId, + notification_category_id: notificationCategoryId, }, callbackUrl: undefined, }); diff --git a/apps/backend/src/lib/upstash.tsx b/apps/backend/src/lib/upstash.tsx index 9744ce7fc9..6b4f48fec8 100644 --- a/apps/backend/src/lib/upstash.tsx +++ b/apps/backend/src/lib/upstash.tsx @@ -19,8 +19,13 @@ export async function ensureUpstashSignature(fullReq: SmartRequest): Promise = globalVar.__stack_neon_prisma_clients ??= new Map(); function getNeonPrismaClient(connectionString: string) { - let neonPrismaClient = prismaClientsStore.neon.get(connectionString); + let neonPrismaClient = neonPrismaClientsStore.get(connectionString); if (!neonPrismaClient) { const schema = getSchemaFromConnectionString(connectionString); const adapter = new PrismaNeon({ connectionString }, { schema }); @@ -74,8 +71,12 @@ export async function getPrismaSchemaForTenancy(tenancy: Tenancy) { } +const postgresPrismaClientsStore: Map = globalVar.__stack_postgres_prisma_clients ??= new Map(); function getPostgresPrismaClient(connectionString: string) { - let postgresPrismaClient = prismaClientsStore.postgres.get(connectionString); + let postgresPrismaClient = postgresPrismaClientsStore.get(connectionString); if (!postgresPrismaClient) { const schema = getSchemaFromConnectionString(connectionString); const adapter = new PrismaPg({ connectionString }, schema ? { schema } : undefined); @@ -83,11 +84,51 @@ function getPostgresPrismaClient(connectionString: string) { client: new PrismaClient({ adapter }), schema, }; - prismaClientsStore.postgres.set(connectionString, postgresPrismaClient); + postgresPrismaClientsStore.set(connectionString, postgresPrismaClient); } return postgresPrismaClient; } +async function tcpPing(host: string, port: number, timeout = 2000) { + return await new Promise((resolve) => { + const s = net.connect({ host, port }).setTimeout(timeout); + + const done = (result: boolean) => { + s.destroy(); + resolve(result); + }; + + s.on("connect", () => done(true)); + s.on("timeout", () => done(false)); + s.on("error", () => done(false)); + }); +} + +const originalGlobalConnectionString = getEnvVariable("STACK_DATABASE_CONNECTION_STRING", ""); +let actualGlobalConnectionString: string = globalVar.__stack_actual_global_connection_string ??= await (async () => { + // If we are on a Mac with OrbStack installed, it's much much faster to use the OrbStack-provided domain instead of + // the container's port forwarding. + // + // For this reason, we check whether we can connect to the database using the OrbStack-provided domain, and if so, + // we use it instead of the original connection string. + if (getNodeEnvironment() === 'development' && process.platform === 'darwin') { + const match = originalGlobalConnectionString.match(/^postgres:\/\/postgres:(.*)@localhost:(\d\d)28\/(.*)$/); + if (match) { + const [, password, portPrefix, schema] = match; + const orbStackDomain = `db.stack-dependencies-${portPrefix}.orb.local`; + const now = performance.now(); + const ok = await tcpPing(orbStackDomain, 5432, 50); // extremely short timeout; OrbStack should be fast to respond, otherwise why are we doing this? + if (ok) { + return `postgres://postgres:${password}@${orbStackDomain}:5432/${schema}`; + } + } + } + return originalGlobalConnectionString; +})(); + + +export const { client: globalPrismaClient, schema: globalPrismaSchema } = getPostgresPrismaClient(actualGlobalConnectionString); + export async function getPrismaClientForSourceOfTruth(sourceOfTruth: CompleteConfig["sourceOfTruth"], branchId: string) { switch (sourceOfTruth.type) { case 'neon': { diff --git a/apps/backend/src/route-handlers/smart-route-handler.tsx b/apps/backend/src/route-handlers/smart-route-handler.tsx index 8acc995c2f..47781eb537 100644 --- a/apps/backend/src/route-handlers/smart-route-handler.tsx +++ b/apps/backend/src/route-handlers/smart-route-handler.tsx @@ -102,13 +102,15 @@ export function handleApiRequest(handler: (req: NextRequest, options: any, reque } // request duration warning - const warnAfterSeconds = 12; - runAsynchronously(async () => { - await wait(warnAfterSeconds * 1000); - if (!hasRequestFinished) { - captureError("request-timeout-watcher", new Error(`Request with ID ${requestId} to endpoint ${req.nextUrl.pathname} has been running for ${warnAfterSeconds} seconds. Try to keep requests short. The request may be cancelled by the serverless provider if it takes too long.`)); - } - }); + if (req.nextUrl.pathname !== "/api/latest/internal/email-queue-step") { + const warnAfterSeconds = 12; + runAsynchronously(async () => { + await wait(warnAfterSeconds * 1000); + if (!hasRequestFinished) { + captureError("request-timeout-watcher", new Error(`Request with ID ${requestId} to endpoint ${req.nextUrl.pathname} has been running for ${warnAfterSeconds} seconds. Try to keep requests short. The request may be cancelled by the serverless provider if it takes too long.`)); + } + }); + } if (!disableExtendedLogging) console.log(`[API REQ] [${requestId}] ${req.method} ${censoredUrl}`); const timeStart = performance.now(); diff --git a/apps/backend/src/utils/telemetry.tsx b/apps/backend/src/utils/telemetry.tsx index 152bef3ae4..021faca6fa 100644 --- a/apps/backend/src/utils/telemetry.tsx +++ b/apps/backend/src/utils/telemetry.tsx @@ -6,7 +6,7 @@ const tracer = trace.getTracer('stack-backend'); export function withTraceSpan

(optionsOrDescription: string | { description: string, attributes?: Record }, fn: (...args: P) => Promise): (...args: P) => Promise { return async (...args: P) => { - return await traceSpan(optionsOrDescription, (span) => fn(...args)); + return await traceSpan(optionsOrDescription, async (span) => await fn(...args)); }; } diff --git a/apps/dashboard/tsconfig.json b/apps/dashboard/tsconfig.json index 5d05b2a1e7..fb9c090878 100644 --- a/apps/dashboard/tsconfig.json +++ b/apps/dashboard/tsconfig.json @@ -14,7 +14,7 @@ "moduleResolution": "bundler", "resolveJsonModule": true, "isolatedModules": true, - "jsx": "react-jsx", + "jsx": "preserve", "incremental": true, "noErrorTruncation": true, "plugins": [ diff --git a/apps/dev-launchpad/public/index.html b/apps/dev-launchpad/public/index.html index 8e89978e53..caea643333 100644 --- a/apps/dev-launchpad/public/index.html +++ b/apps/dev-launchpad/public/index.html @@ -286,6 +286,14 @@

Background services

importance: 1, img: "https://cdn.prod.website-files.com/655b60964be1a1b36c746790/655b60964be1a1b36c746d41_646dfce3b9c4849f6e401bff_supabase-logo-icon_1.png", }, + { + name: "Drizzle Gateway", + portSuffix: "32", + description: [ + "Manage Drizzle configs", + ], + importance: 1, + }, { name: "JS example", portSuffix: "19", diff --git a/apps/e2e/tests/backend/backend-helpers.ts b/apps/e2e/tests/backend/backend-helpers.ts index 857f9a332b..141cc60c9f 100644 --- a/apps/e2e/tests/backend/backend-helpers.ts +++ b/apps/e2e/tests/backend/backend-helpers.ts @@ -365,10 +365,17 @@ export namespace Auth { "headers": Headers {