diff --git a/docs/docker.md b/docs/docker.md index 8262b13d..0f3b8213 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -50,7 +50,7 @@ Set your webhook endpoint in the Stripe dashboard to point to your server’s `/ | `AUTO_EXPAND_LISTS` | Fetch all list items from Stripe (default: false) | No | | `BACKFILL_RELATED_ENTITIES` | Backfill related entities for foreign key integrity (default: true) | No | | `MAX_POSTGRES_CONNECTIONS` | Max Postgres connection pool size (default: 10) | No | -| `REVALIDATE_ENTITY_VIA_STRIPE_API` | Always fetch latest entity from Stripe (default: false) | No | +| `REVALIDATE_OBJECTS_VIA_STRIPE_API` | Always fetch latest entity from Stripe (default: false) | No | ## Endpoints diff --git a/docs/typescript.md b/docs/typescript.md index fbaea759..0f3f06da 100644 --- a/docs/typescript.md +++ b/docs/typescript.md @@ -36,18 +36,18 @@ await sync.processWebhook(payload, signature) ## Configuration -| Option | Type | Description | -| ------------------------------ | ------- | -------------------------------------------------------------------------- | -| `databaseUrl` | string | Postgres connection string | -| `schema` | string | Database schema name (default: `stripe`) | -| `stripeSecretKey` | string | Stripe secret key | -| `stripeWebhookSecret` | string | Stripe webhook signing secret | -| `stripeApiVersion` | string | Stripe API version (default: `2020-08-27`) | -| `autoExpandLists` | boolean | Fetch all list items from Stripe (not just the default 10) | -| `backfillRelatedEntities` | boolean | Ensure related entities are present for foreign key integrity | -| `revalidateEntityViaStripeApi` | boolean | Always fetch latest entity from Stripe instead of trusting webhook payload | -| `maxPostgresConnections` | number | Maximum Postgres connections | -| `logger` | Logger | Logger instance (pino) | +| Option | Type | Description | +| ------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `databaseUrl` | string | Postgres connection string | +| `schema` | string | Database schema name (default: `stripe`) | +| `stripeSecretKey` | string | Stripe secret key | +| `stripeWebhookSecret` | string | Stripe webhook signing secret | +| `stripeApiVersion` | string | Stripe API version (default: `2020-08-27`) | +| `autoExpandLists` | boolean | Fetch all list items from Stripe (not just the default 10) | +| `backfillRelatedEntities` | boolean | Ensure related entities are present for foreign key integrity | +| `revalidateObjectsViaStripeApi` | Array | Always fetch latest entity from Stripe instead of trusting webhook payload, possible values: charge, credit_note, customer, dispute, invoice, payment_intent, payment_method, plan, price, product, refund, review, radar.early_fraud_warning, setup_intent, subscription, subscription_schedule, tax_id | +| `maxPostgresConnections` | number | Maximum Postgres connections | +| `logger` | Logger | Logger instance (pino) | ## Database Schema diff --git a/packages/fastify-app/.env.sample b/packages/fastify-app/.env.sample index c971d7dd..0a37f6e3 100644 --- a/packages/fastify-app/.env.sample +++ b/packages/fastify-app/.env.sample @@ -34,5 +34,5 @@ BACKFILL_RELATED_ENTITIES=true MAX_POSTGRES_CONNECTIONS=20 # If true, the webhook data is not used and instead the webhook is just a trigger to fetch the entity from Stripe again. This ensures that a race condition with failed webhooks can never accidentally overwrite the data with an older state. -# Default: false -REVALIDATE_ENTITY_VIA_STRIPE_API=false \ No newline at end of file +# Default: +REVALIDATE_OBJECTS_VIA_STRIPE_API=payment_intent,invoice,customer,subscription diff --git a/packages/fastify-app/README.md b/packages/fastify-app/README.md index cd454136..c805cfa6 100644 --- a/packages/fastify-app/README.md +++ b/packages/fastify-app/README.md @@ -38,19 +38,19 @@ Set your webhook endpoint in the Stripe dashboard to point to your server’s `/ ## Environment Variables -| Variable | Description | Required | -| ---------------------------------- | ------------------------------------------------------------------- | -------- | -| `DATABASE_URL` | Postgres connection string (with `search_path=stripe`) | Yes | -| `STRIPE_WEBHOOK_SECRET` | Stripe webhook signing secret | Yes | -| `API_KEY` | API key for admin endpoints (backfilling, etc.) | Yes | -| `SCHEMA` | Database schema name (default: `stripe`) | No | -| `STRIPE_SECRET_KEY` | Stripe secret key (needed for active sync/backfill) | No | -| `PORT` | Port to run the server on (default: 8080) | No | -| `STRIPE_API_VERSION` | Stripe API version (default: `2020-08-27`) | No | -| `AUTO_EXPAND_LISTS` | Fetch all list items from Stripe (default: false) | No | -| `BACKFILL_RELATED_ENTITIES` | Backfill related entities for foreign key integrity (default: true) | No | -| `MAX_POSTGRES_CONNECTIONS` | Max Postgres connection pool size (default: 10) | No | -| `REVALIDATE_ENTITY_VIA_STRIPE_API` | Always fetch latest entity from Stripe (default: false) | No | +| Variable | Description | Required | +| ----------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| `DATABASE_URL` | Postgres connection string (with `search_path=stripe`) | Yes | +| `STRIPE_WEBHOOK_SECRET` | Stripe webhook signing secret | Yes | +| `API_KEY` | API key for admin endpoints (backfilling, etc.) | Yes | +| `SCHEMA` | Database schema name (default: `stripe`) | No | +| `STRIPE_SECRET_KEY` | Stripe secret key (needed for active sync/backfill) | No | +| `PORT` | Port to run the server on (default: 8080) | No | +| `STRIPE_API_VERSION` | Stripe API version (default: `2020-08-27`) | No | +| `AUTO_EXPAND_LISTS` | Fetch all list items from Stripe (default: false) | No | +| `BACKFILL_RELATED_ENTITIES` | Backfill related entities for foreign key integrity (default: true) | No | +| `MAX_POSTGRES_CONNECTIONS` | Max Postgres connection pool size (default: 10) | No | +| `REVALIDATE_OBJECTS_VIA_STRIPE_API` | Always fetch latest entity from Stripe instead of trusting webhook payload, possible values: charge, credit_note, customer, dispute, invoice, payment_intent, payment_method, plan, price, product, refund, review, radar.early_fraud_warning, setup_intent, subscription, subscription_schedule, tax_id | No | ## Endpoints diff --git a/packages/fastify-app/src/test/revalidate.test.ts b/packages/fastify-app/src/test/revalidate.test.ts new file mode 100644 index 00000000..ef791fa1 --- /dev/null +++ b/packages/fastify-app/src/test/revalidate.test.ts @@ -0,0 +1,39 @@ +import { StripeSync } from '@supabase/stripe-sync-engine' +import { vitest, beforeAll, describe, test, expect } from 'vitest' +import { runMigrations } from '@supabase/stripe-sync-engine' +import { getConfig } from '../utils/config' +import { mockStripe } from './helpers/mockStripe' +import { logger } from '../logger' +import type Stripe from 'stripe' + +let stripeSync: StripeSync + +beforeAll(async () => { + process.env.REVALIDATE_OBJECTS_VIA_STRIPE_API = 'invoice' + + const config = getConfig() + await runMigrations({ + databaseUrl: config.databaseUrl, + schema: config.schema, + logger, + }) + + stripeSync = new StripeSync(config) + const stripe = Object.assign(stripeSync.stripe, mockStripe) + vitest.spyOn(stripeSync, 'stripe', 'get').mockReturnValue(stripe) +}) + +describe('invoices', () => { + test('should revalidate entity if enabled', async () => { + const eventBody = await import(`./stripe/invoice_paid.json`).then( + ({ default: myData }) => myData + ) + + await stripeSync.processEvent(eventBody as unknown as Stripe.Event) + + const result = await stripeSync.postgresClient.query( + `select customer from stripe.invoices where id = 'in_1KJqKBJDPojXS6LNJbvLUgEy' limit 1` + ) + expect(result.rows[0].customer).toEqual('cus_J7Mkgr8mvbl1eK') // from stripe mock + }) +}) diff --git a/packages/fastify-app/src/utils/config.ts b/packages/fastify-app/src/utils/config.ts index 16c1adc2..9938d28b 100644 --- a/packages/fastify-app/src/utils/config.ts +++ b/packages/fastify-app/src/utils/config.ts @@ -1,3 +1,4 @@ +import type { RevalidateEntity } from '@supabase/stripe-sync-engine' import { config } from 'dotenv' function getConfigFromEnv(key: string, defaultValue?: string): string { @@ -41,7 +42,7 @@ export type StripeSyncServerConfig = { maxPostgresConnections?: number - revalidateEntityViaStripeApi: boolean + revalidateObjectsViaStripeApi: Array port: number } @@ -60,7 +61,9 @@ export function getConfig(): StripeSyncServerConfig { autoExpandLists: getConfigFromEnv('AUTO_EXPAND_LISTS', 'false') === 'true', backfillRelatedEntities: getConfigFromEnv('BACKFILL_RELATED_ENTITIES', 'true') === 'true', maxPostgresConnections: Number(getConfigFromEnv('MAX_POSTGRES_CONNECTIONS', '10')), - revalidateEntityViaStripeApi: - getConfigFromEnv('REVALIDATE_ENTITY_VIA_STRIPE_API', 'false') === 'true', + revalidateObjectsViaStripeApi: getConfigFromEnv('REVALIDATE_OBJECTS_VIA_STRIPE_API', '') + .split(',') + .map((it) => it.trim()) + .filter((it) => it.length > 0) as Array, } } diff --git a/packages/sync-engine/README.md b/packages/sync-engine/README.md index 3867fc80..98129092 100644 --- a/packages/sync-engine/README.md +++ b/packages/sync-engine/README.md @@ -39,18 +39,18 @@ await sync.processWebhook(payload, signature) ## Configuration -| Option | Type | Description | -| ------------------------------ | ------- | -------------------------------------------------------------------------- | -| `databaseUrl` | string | Postgres connection string | -| `schema` | string | Database schema name (default: `stripe`) | -| `stripeSecretKey` | string | Stripe secret key | -| `stripeWebhookSecret` | string | Stripe webhook signing secret | -| `stripeApiVersion` | string | Stripe API version (default: `2020-08-27`) | -| `autoExpandLists` | boolean | Fetch all list items from Stripe (not just the default 10) | -| `backfillRelatedEntities` | boolean | Ensure related entities are present for foreign key integrity | -| `revalidateEntityViaStripeApi` | boolean | Always fetch latest entity from Stripe instead of trusting webhook payload | -| `maxPostgresConnections` | number | Maximum Postgres connections | -| `logger` | Logger | Logger instance (pino) | +| Option | Type | Description | +| ------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `databaseUrl` | string | Postgres connection string | +| `schema` | string | Database schema name (default: `stripe`) | +| `stripeSecretKey` | string | Stripe secret key | +| `stripeWebhookSecret` | string | Stripe webhook signing secret | +| `stripeApiVersion` | string | Stripe API version (default: `2020-08-27`) | +| `autoExpandLists` | boolean | Fetch all list items from Stripe (not just the default 10) | +| `backfillRelatedEntities` | boolean | Ensure related entities are present for foreign key integrity | +| `revalidateObjectsViaStripeApi` | Array | Always fetch latest entity from Stripe instead of trusting webhook payload, possible values: charge, credit_note, customer, dispute, invoice, payment_intent, payment_method, plan, price, product, refund, review, radar.early_fraud_warning, setup_intent, subscription, subscription_schedule, tax_id | +| `maxPostgresConnections` | number | Maximum Postgres connections | +| `logger` | Logger | Logger instance (pino) | ## Database Schema diff --git a/packages/sync-engine/src/stripeSync.ts b/packages/sync-engine/src/stripeSync.ts index 714814b3..395fcf93 100644 --- a/packages/sync-engine/src/stripeSync.ts +++ b/packages/sync-engine/src/stripeSync.ts @@ -16,7 +16,13 @@ import { taxIdSchema } from './schemas/tax_id' import { subscriptionItemSchema } from './schemas/subscription_item' import { subscriptionScheduleSchema } from './schemas/subscription_schedules' import { subscriptionSchema } from './schemas/subscription' -import { StripeSyncConfig, Sync, SyncBackfill, SyncBackfillParams } from './types' +import { + StripeSyncConfig, + Sync, + SyncBackfill, + SyncBackfillParams, + type RevalidateEntity, +} from './types' import { earlyFraudWarningSchema } from './schemas/early_fraud_warning' import { reviewSchema } from './schemas/review' import { refundSchema } from './schemas/refund' @@ -66,8 +72,10 @@ export class StripeSync { this.config.stripeWebhookSecret ) - const syncTimestamp = new Date(event.created * 1000).toISOString() + return this.processEvent(event) + } + async processEvent(event: Stripe.Event) { switch (event.type) { case 'charge.captured': case 'charge.expired': @@ -84,7 +92,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for charge ${charge.id}` ) - await this.upsertCharges([charge], false, syncTimestamp) + await this.upsertCharges([charge], false, this.getSyncTimestamp(event)) break } case 'customer.deleted': { @@ -98,7 +106,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for customer ${customer.id}` ) - await this.upsertCustomers([customer], syncTimestamp) + await this.upsertCustomers([customer], this.getSyncTimestamp(event)) break } case 'customer.created': @@ -112,7 +120,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for customer ${customer.id}` ) - await this.upsertCustomers([customer], syncTimestamp) + await this.upsertCustomers([customer], this.getSyncTimestamp(event)) break } case 'customer.subscription.created': @@ -132,7 +140,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for subscription ${subscription.id}` ) - await this.upsertSubscriptions([subscription], false, syncTimestamp) + await this.upsertSubscriptions([subscription], false, this.getSyncTimestamp(event)) break } case 'customer.tax_id.updated': @@ -145,7 +153,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for taxId ${taxId.id}` ) - await this.upsertTaxIds([taxId], false, syncTimestamp) + await this.upsertTaxIds([taxId], false, this.getSyncTimestamp(event)) break } case 'customer.tax_id.deleted': { @@ -180,7 +188,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for invoice ${invoice.id}` ) - await this.upsertInvoices([invoice], false, syncTimestamp) + await this.upsertInvoices([invoice], false, this.getSyncTimestamp(event)) break } case 'product.created': @@ -195,7 +203,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for product ${product.id}` ) - await this.upsertProducts([product], syncTimestamp) + await this.upsertProducts([product], this.getSyncTimestamp(event)) } catch (err) { if (err instanceof Stripe.errors.StripeAPIError && err.code === 'resource_missing') { await this.deleteProduct(event.data.object.id) @@ -227,7 +235,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for price ${price.id}` ) - await this.upsertPrices([price], false, syncTimestamp) + await this.upsertPrices([price], false, this.getSyncTimestamp(event)) } catch (err) { if (err instanceof Stripe.errors.StripeAPIError && err.code === 'resource_missing') { await this.deletePrice(event.data.object.id) @@ -259,7 +267,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for plan ${plan.id}` ) - await this.upsertPlans([plan], false, syncTimestamp) + await this.upsertPlans([plan], false, this.getSyncTimestamp(event)) } catch (err) { if (err instanceof Stripe.errors.StripeAPIError && err.code === 'resource_missing') { await this.deletePlan(event.data.object.id) @@ -292,7 +300,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for setupIntent ${setupIntent.id}` ) - await this.upsertSetupIntents([setupIntent], false, syncTimestamp) + await this.upsertSetupIntents([setupIntent], false, this.getSyncTimestamp(event)) break } case 'subscription_schedule.aborted': @@ -311,7 +319,11 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for subscriptionSchedule ${subscriptionSchedule.id}` ) - await this.upsertSubscriptionSchedules([subscriptionSchedule], false, syncTimestamp) + await this.upsertSubscriptionSchedules( + [subscriptionSchedule], + false, + this.getSyncTimestamp(event) + ) break } case 'payment_method.attached': @@ -327,7 +339,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for paymentMethod ${paymentMethod.id}` ) - await this.upsertPaymentMethods([paymentMethod], false, syncTimestamp) + await this.upsertPaymentMethods([paymentMethod], false, this.getSyncTimestamp(event)) break } case 'charge.dispute.created': @@ -344,7 +356,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for dispute ${dispute.id}` ) - await this.upsertDisputes([dispute], false, syncTimestamp) + await this.upsertDisputes([dispute], false, this.getSyncTimestamp(event)) break } case 'payment_intent.amount_capturable_updated': @@ -364,7 +376,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for paymentIntent ${paymentIntent.id}` ) - await this.upsertPaymentIntents([paymentIntent], false, syncTimestamp) + await this.upsertPaymentIntents([paymentIntent], false, this.getSyncTimestamp(event)) break } @@ -380,7 +392,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for creditNote ${creditNote.id}` ) - await this.upsertCreditNotes([creditNote], false, syncTimestamp) + await this.upsertCreditNotes([creditNote], false, this.getSyncTimestamp(event)) break } @@ -395,7 +407,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for earlyFraudWarning ${earlyFraudWarning.id}` ) - await this.upsertEarlyFraudWarning([earlyFraudWarning], false, syncTimestamp) + await this.upsertEarlyFraudWarning([earlyFraudWarning], false, this.getSyncTimestamp(event)) break } @@ -412,7 +424,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for refund ${refund.id}` ) - await this.upsertRefunds([refund], false, syncTimestamp) + await this.upsertRefunds([refund], false, this.getSyncTimestamp(event)) break } @@ -426,7 +438,7 @@ export class StripeSync { `Received webhook ${event.id}: ${event.type} for review ${review.id}` ) - await this.upsertReviews([review], false, syncTimestamp) + await this.upsertReviews([review], false, this.getSyncTimestamp(event)) break } @@ -436,13 +448,23 @@ export class StripeSync { } } - private async fetchOrUseWebhookData( + private getSyncTimestamp(event: Stripe.Event) { + return this.shouldRefetchEntity(event.data.object) + ? new Date().toISOString() + : new Date(event.created * 1000).toISOString() + } + + private shouldRefetchEntity(entity: { object: string }) { + return this.config.revalidateObjectsViaStripeApi?.includes(entity.object as RevalidateEntity) + } + + private async fetchOrUseWebhookData( entity: T, fetchFn: (id: string) => Promise ): Promise { if (!entity.id) return entity - if (this.config.revalidateEntityViaStripeApi) { + if (this.shouldRefetchEntity(entity)) { return fetchFn(entity.id) } diff --git a/packages/sync-engine/src/types.ts b/packages/sync-engine/src/types.ts index 8b6c7767..d490dc6b 100644 --- a/packages/sync-engine/src/types.ts +++ b/packages/sync-engine/src/types.ts @@ -1,5 +1,24 @@ import pino from 'pino' +export type RevalidateEntity = + | 'charge' + | 'credit_note' + | 'customer' + | 'dispute' + | 'invoice' + | 'payment_intent' + | 'payment_method' + | 'plan' + | 'price' + | 'product' + | 'refund' + | 'review' + | 'radar.early_fraud_warning' + | 'setup_intent' + | 'subscription' + | 'subscription_schedule' + | 'tax_id' + export type StripeSyncConfig = { /** Postgres database URL including authentication */ databaseUrl: string @@ -33,7 +52,7 @@ export type StripeSyncConfig = { * * Default: false */ - revalidateEntityViaStripeApi?: boolean + revalidateObjectsViaStripeApi?: Array maxPostgresConnections?: number