diff --git a/docs/database/postgres.mdx b/docs/database/postgres.mdx index 4224066b346..518e38a27d4 100644 --- a/docs/database/postgres.mdx +++ b/docs/database/postgres.mdx @@ -80,6 +80,7 @@ export default buildConfig({ | `afterSchemaInit` | Drizzle schema hook. Runs after the schema is built. [More Details](#afterschemainit) | | `generateSchemaOutputFile` | Override generated schema from `payload generate:db-schema` file path. Defaults to `{CWD}/src/payload-generated.schema.ts` | | `allowIDOnCreate` | Set to `true` to use the `id` passed in data on the create API operations without using a custom ID field. | +| `readReplicas` | An array of DB read replicas connection strings, can be used to offload read-heavy traffic. | ## Access to Drizzle diff --git a/packages/db-postgres/src/connect.ts b/packages/db-postgres/src/connect.ts index 3b11996d0cf..49236862492 100644 --- a/packages/db-postgres/src/connect.ts +++ b/packages/db-postgres/src/connect.ts @@ -1,31 +1,32 @@ import type { DrizzleAdapter } from '@payloadcms/drizzle/types' -import type { Connect, Migration, Payload } from 'payload' +import type { Connect, Migration } from 'payload' import { pushDevSchema } from '@payloadcms/drizzle' import { drizzle } from 'drizzle-orm/node-postgres' +import { withReplicas } from 'drizzle-orm/pg-core' import type { PostgresAdapter } from './types.js' const connectWithReconnect = async function ({ adapter, - payload, + pool, reconnect = false, }: { adapter: PostgresAdapter - payload: Payload + pool: PostgresAdapter['pool'] reconnect?: boolean }) { let result if (!reconnect) { - result = await adapter.pool.connect() + result = await pool.connect() } else { try { - result = await adapter.pool.connect() + result = await pool.connect() } catch (ignore) { setTimeout(() => { - payload.logger.info('Reconnecting to postgres') - void connectWithReconnect({ adapter, payload, reconnect: true }) + adapter.payload.logger.info('Reconnecting to postgres') + void connectWithReconnect({ adapter, pool, reconnect: true }) }, 1000) } } @@ -35,7 +36,7 @@ const connectWithReconnect = async function ({ result.prependListener('error', (err) => { try { if (err.code === 'ECONNRESET') { - void connectWithReconnect({ adapter, payload, reconnect: true }) + void connectWithReconnect({ adapter, pool, reconnect: true }) } } catch (ignore) { // swallow error @@ -54,12 +55,29 @@ export const connect: Connect = async function connect( try { if (!this.pool) { this.pool = new this.pg.Pool(this.poolOptions) - await connectWithReconnect({ adapter: this, payload: this.payload }) + await connectWithReconnect({ adapter: this, pool: this.pool }) } const logger = this.logger || false this.drizzle = drizzle({ client: this.pool, logger, schema: this.schema }) + if (this.readReplicaOptions) { + const readReplicas = this.readReplicaOptions.map((connectionString) => { + const options = { + ...this.poolOptions, + connectionString, + } + const pool = new this.pg.Pool(options) + void connectWithReconnect({ + adapter: this, + pool, + }) + return drizzle({ client: pool, logger, schema: this.schema }) + }) + const myReplicas = withReplicas(this.drizzle, readReplicas as any) + this.drizzle = myReplicas + } + if (!hotReload) { if (process.env.PAYLOAD_DROP_DATABASE === 'true') { this.payload.logger.info(`---- DROPPING TABLES SCHEMA(${this.schemaName || 'public'}) ----`) diff --git a/packages/db-postgres/src/index.ts b/packages/db-postgres/src/index.ts index 8d9597c5109..fc372681745 100644 --- a/packages/db-postgres/src/index.ts +++ b/packages/db-postgres/src/index.ts @@ -139,6 +139,7 @@ export function postgresAdapter(args: Args): DatabaseAdapterObj prodMigrations: args.prodMigrations, // @ts-expect-error - vestiges of when tsconfig was not strict. Feel free to improve push: args.push, + readReplicaOptions: args.readReplicas, relations: {}, relationshipsSuffix: args.relationshipsSuffix || '_rels', schema: {}, diff --git a/packages/db-postgres/src/types.ts b/packages/db-postgres/src/types.ts index 8d19181cf2a..59ec30357ea 100644 --- a/packages/db-postgres/src/types.ts +++ b/packages/db-postgres/src/types.ts @@ -3,13 +3,19 @@ import type { GenericEnum, MigrateDownArgs, MigrateUpArgs, - PostgresDB, PostgresSchemaHook, } from '@payloadcms/drizzle/postgres' import type { DrizzleAdapter } from '@payloadcms/drizzle/types' -import type { DrizzleConfig } from 'drizzle-orm' +import type { DrizzleConfig, ExtractTablesWithRelations } from 'drizzle-orm' import type { NodePgDatabase } from 'drizzle-orm/node-postgres' -import type { PgSchema, PgTableFn, PgTransactionConfig } from 'drizzle-orm/pg-core' +import type { + PgDatabase, + PgQueryResultHKT, + PgSchema, + PgTableFn, + PgTransactionConfig, + PgWithReplicas, +} from 'drizzle-orm/pg-core' import type { Pool, PoolConfig } from 'pg' type PgDependency = typeof import('pg') @@ -55,6 +61,7 @@ export type Args = { up: (args: MigrateUpArgs) => Promise }[] push?: boolean + readReplicas?: string[] relationshipsSuffix?: string /** * The schema name to use for the database @@ -74,7 +81,16 @@ type ResolveSchemaType = 'schema' extends keyof T ? T['schema'] : GeneratedDatabaseSchema['schemaUntyped'] -type Drizzle = NodePgDatabase> +type Drizzle = + | NodePgDatabase> + | PgWithReplicas< + PgDatabase< + PgQueryResultHKT, + Record, + ExtractTablesWithRelations> + > + > + export type PostgresAdapter = { drizzle: Drizzle pg: PgDependency diff --git a/packages/db-vercel-postgres/src/connect.ts b/packages/db-vercel-postgres/src/connect.ts index 9ca43d97b90..d74f00c265b 100644 --- a/packages/db-vercel-postgres/src/connect.ts +++ b/packages/db-vercel-postgres/src/connect.ts @@ -4,6 +4,7 @@ import type { Connect, Migration } from 'payload' import { pushDevSchema } from '@payloadcms/drizzle' import { sql, VercelPool } from '@vercel/postgres' import { drizzle } from 'drizzle-orm/node-postgres' +import { withReplicas } from 'drizzle-orm/pg-core' import pg from 'pg' import type { VercelPostgresAdapter } from './types.js' @@ -46,6 +47,19 @@ export const connect: Connect = async function connect( schema: this.schema, }) + if (this.readReplicaOptions) { + const readReplicas = this.readReplicaOptions.map((connectionString) => { + const options = { + ...this.poolOptions, + connectionString, + } + const pool = new VercelPool(options) + return drizzle({ client: pool, logger, schema: this.schema }) + }) + const myReplicas = withReplicas(this.drizzle, readReplicas as any) + this.drizzle = myReplicas + } + if (!hotReload) { if (process.env.PAYLOAD_DROP_DATABASE === 'true') { this.payload.logger.info(`---- DROPPING TABLES SCHEMA(${this.schemaName || 'public'}) ----`) diff --git a/packages/db-vercel-postgres/src/index.ts b/packages/db-vercel-postgres/src/index.ts index f4ed3aada33..bb6e2ecaf51 100644 --- a/packages/db-vercel-postgres/src/index.ts +++ b/packages/db-vercel-postgres/src/index.ts @@ -174,6 +174,7 @@ export function vercelPostgresAdapter(args: Args = {}): DatabaseAdapterObj Promise }[] push?: boolean + readReplicas?: string[] relationshipsSuffix?: string /** * The schema name to use for the database diff --git a/packages/drizzle/src/postgres/types.ts b/packages/drizzle/src/postgres/types.ts index 7a6f6ed410c..696d13797df 100644 --- a/packages/drizzle/src/postgres/types.ts +++ b/packages/drizzle/src/postgres/types.ts @@ -159,6 +159,7 @@ export type BasePostgresAdapter = { up: (args: MigrateUpArgs) => Promise }[] push: boolean + readReplicaOptions?: string[] rejectInitializing: () => void relations: Record relationshipsSuffix?: string diff --git a/test/database/pg-replica/docker-compose.yaml b/test/database/pg-replica/docker-compose.yaml new file mode 100644 index 00000000000..713d0acb07c --- /dev/null +++ b/test/database/pg-replica/docker-compose.yaml @@ -0,0 +1,36 @@ +# Copyright Broadcom, Inc. All Rights Reserved. +# SPDX-License-Identifier: APACHE-2.0 + +services: + postgresql-master: + image: docker.io/bitnami/postgresql:17 + ports: + - '5433:5432' + volumes: + - 'postgresql_master_data:/bitnami/postgresql' + environment: + - POSTGRESQL_REPLICATION_MODE=master + - POSTGRESQL_REPLICATION_USER=repl_user + - POSTGRESQL_REPLICATION_PASSWORD=repl_password + - POSTGRESQL_USERNAME=postgres + - POSTGRESQL_PASSWORD=my_password + - POSTGRESQL_DATABASE=my_database + - ALLOW_EMPTY_PASSWORD=yes + postgresql-slave: + image: docker.io/bitnami/postgresql:17 + ports: + - '5434:5432' + depends_on: + - postgresql-master + environment: + - POSTGRESQL_REPLICATION_MODE=slave + - POSTGRESQL_REPLICATION_USER=repl_user + - POSTGRESQL_REPLICATION_PASSWORD=repl_password + - POSTGRESQL_MASTER_HOST=postgresql-master + - POSTGRESQL_PASSWORD=my_password + - POSTGRESQL_MASTER_PORT_NUMBER=5432 + - ALLOW_EMPTY_PASSWORD=yes + +volumes: + postgresql_master_data: + driver: local diff --git a/test/generateDatabaseAdapter.ts b/test/generateDatabaseAdapter.ts index a71641bf9df..5d28069b8b2 100644 --- a/test/generateDatabaseAdapter.ts +++ b/test/generateDatabaseAdapter.ts @@ -47,6 +47,26 @@ export const allDatabaseAdapters = { connectionString: process.env.POSTGRES_URL || 'postgres://127.0.0.1:5432/payloadtests', }, })`, + 'postgres-read-replica': ` + import { postgresAdapter } from '@payloadcms/db-postgres' + + export const databaseAdapter = postgresAdapter({ + pool: { + connectionString: process.env.POSTGRES_URL, + }, + readReplicas: [process.env.POSTGRES_REPLICA_URL], + }) + `, + 'vercel-postgres-read-replica': ` + import { vercelPostgresAdapter } from '@payloadcms/db-vercel-postgres' + + export const databaseAdapter = vercelPostgresAdapter({ + pool: { + connectionString: process.env.POSTGRES_URL, + }, + readReplicas: [process.env.POSTGRES_REPLICA_URL], + }) + `, sqlite: ` import { sqliteAdapter } from '@payloadcms/db-sqlite'