feat(db-*): allows for running migrations in production automatically (#7563)

jmikrut · web-flow · commit 1cb1e5e8b3ee · 2024-08-07T13:57:12.000-04:00
## Description

Introduces a pattern for running migrations upon Payload init in
production.
diff --git a/packages/db-mongodb/src/connect.ts b/packages/db-mongodb/src/connect.ts
@@ -54,6 +54,10 @@ export const connect: Connect = async function connect(
         this.payload.logger.info('---- DROPPED DATABASE ----')
       }
     }
+
+    if (process.env.NODE_ENV === 'production' && this.prodMigrations) {
+      await this.migrate({ migrations: this.prodMigrations })
+    }
   } catch (err) {
     console.log(err)
     this.payload.logger.error(`Error: cannot connect to MongoDB. Details: ${err.message}`, err)
diff --git a/packages/db-mongodb/src/index.ts b/packages/db-mongodb/src/index.ts
@@ -8,7 +8,7 @@ import mongoose from 'mongoose'
 import path from 'path'
 import { createDatabaseAdapter } from 'payload'
 
-import type { CollectionModel, GlobalModel } from './types.js'
+import type { CollectionModel, GlobalModel, MigrateDownArgs, MigrateUpArgs } from './types.js'
 
 import { connect } from './connect.js'
 import { count } from './count.js'
@@ -78,6 +78,11 @@ export interface Args {
    * typed as any to avoid dependency
    */
   mongoMemoryServer?: MongoMemoryReplSet
+  prodMigrations?: {
+    down: (args: MigrateDownArgs) => Promise<void>
+    name: string
+    up: (args: MigrateUpArgs) => Promise<void>
+  }[]
   transactionOptions?: TransactionOptions | false
   /** The URL to connect to MongoDB or false to start payload and prevent connecting */
   url: false | string
@@ -90,6 +95,11 @@ export type MongooseAdapter = {
   connection: Connection
   globals: GlobalModel
   mongoMemoryServer: MongoMemoryReplSet
+  prodMigrations?: {
+    down: (args: MigrateDownArgs) => Promise<void>
+    name: string
+    up: (args: MigrateUpArgs) => Promise<void>
+  }[]
   sessions: Record<number | string, ClientSession>
   versions: {
     [slug: string]: CollectionModel
@@ -107,6 +117,11 @@ declare module 'payload' {
     connection: Connection
     globals: GlobalModel
     mongoMemoryServer: MongoMemoryReplSet
+    prodMigrations?: {
+      down: (args: MigrateDownArgs) => Promise<void>
+      name: string
+      up: (args: MigrateUpArgs) => Promise<void>
+    }[]
     sessions: Record<number | string, ClientSession>
     transactionOptions: TransactionOptions
     versions: {
@@ -121,6 +136,7 @@ export function mongooseAdapter({
   disableIndexHints = false,
   migrationDir: migrationDirArg,
   mongoMemoryServer,
+  prodMigrations,
   transactionOptions = {},
   url,
 }: Args): DatabaseAdapterObj {
@@ -167,6 +183,7 @@ export function mongooseAdapter({
       migrateFresh,
       migrationDir,
       payload,
+      prodMigrations,
       queryDrafts,
       rollbackTransaction,
       updateGlobal,
diff --git a/packages/db-postgres/src/connect.ts b/packages/db-postgres/src/connect.ts
@@ -91,4 +91,8 @@ export const connect: Connect = async function connect(
   }
 
   if (typeof this.resolveInitializing === 'function') this.resolveInitializing()
+
+  if (process.env.NODE_ENV === 'production' && this.prodMigrations) {
+    await this.migrate({ migrations: this.prodMigrations })
+  }
 }
diff --git a/packages/db-postgres/src/createMigration.ts b/packages/db-postgres/src/createMigration.ts
@@ -4,7 +4,7 @@ import type { CreateMigration } from 'payload'
 import fs from 'fs'
 import { createRequire } from 'module'
 import path from 'path'
-import { getPredefinedMigration } from 'payload'
+import { getPredefinedMigration, writeMigrationIndex } from 'payload'
 import prompts from 'prompts'
 import { fileURLToPath } from 'url'
 
@@ -115,5 +115,8 @@ export const createMigration: CreateMigration = async function createMigration(
       upSQL: upSQL || `  // Migration code`,
     }),
   )
+
+  writeMigrationIndex({ migrationsDir: payload.db.migrationDir })
+
   payload.logger.info({ msg: `Migration created at ${filePath}.ts` })
 }
diff --git a/packages/db-postgres/src/index.ts b/packages/db-postgres/src/index.ts
@@ -32,7 +32,7 @@ import {
   updateOne,
   updateVersion,
 } from '@payloadcms/drizzle'
-import { type PgSchema, pgEnum, pgSchema, pgTable } from 'drizzle-orm/pg-core'
+import { pgEnum, pgSchema, pgTable } from 'drizzle-orm/pg-core'
 import { createDatabaseAdapter } from 'payload'
 
 import type { Args, PostgresAdapter } from './types.js'
@@ -94,6 +94,7 @@ export function postgresAdapter(args: Args): DatabaseAdapterObj<PostgresAdapter>
       pgSchema: adapterSchema,
       pool: undefined,
       poolOptions: args.pool,
+      prodMigrations: args.prodMigrations,
       push: args.push,
       relations: {},
       relationshipsSuffix: args.relationshipsSuffix || '_rels',
diff --git a/packages/db-postgres/src/types.ts b/packages/db-postgres/src/types.ts
@@ -33,6 +33,11 @@ export type Args = {
   logger?: DrizzleConfig['logger']
   migrationDir?: string
   pool: PoolConfig
+  prodMigrations?: {
+    down: (args: MigrateDownArgs) => Promise<void>
+    name: string
+    up: (args: MigrateUpArgs) => Promise<void>
+  }[]
   push?: boolean
   relationshipsSuffix?: string
   /**
@@ -136,6 +141,11 @@ export type PostgresAdapter = {
   pgSchema?: Schema
   pool: Pool
   poolOptions: Args['pool']
+  prodMigrations?: {
+    down: (args: MigrateDownArgs) => Promise<void>
+    name: string
+    up: (args: MigrateUpArgs) => Promise<void>
+  }[]
   push: boolean
   rejectInitializing: () => void
   relations: Record<string, GenericRelation>
@@ -178,6 +188,11 @@ declare module 'payload' {
     pgSchema?: { table: PgTableFn } | PgSchema
     pool: Pool
     poolOptions: Args['pool']
+    prodMigrations?: {
+      down: (args: MigrateDownArgs) => Promise<void>
+      name: string
+      up: (args: MigrateUpArgs) => Promise<void>
+    }[]
     push: boolean
     rejectInitializing: () => void
     relationshipsSuffix?: string
diff --git a/packages/db-sqlite/src/connect.ts b/packages/db-sqlite/src/connect.ts
@@ -52,4 +52,8 @@ export const connect: Connect = async function connect(
   }
 
   if (typeof this.resolveInitializing === 'function') this.resolveInitializing()
+
+  if (process.env.NODE_ENV === 'production' && this.prodMigrations) {
+    await this.migrate({ migrations: this.prodMigrations })
+  }
 }
diff --git a/packages/db-sqlite/src/createMigration.ts b/packages/db-sqlite/src/createMigration.ts
@@ -4,7 +4,7 @@ import type { CreateMigration } from 'payload'
 import fs from 'fs'
 import { createRequire } from 'module'
 import path from 'path'
-import { getPredefinedMigration } from 'payload'
+import { getPredefinedMigration, writeMigrationIndex } from 'payload'
 import prompts from 'prompts'
 import { fileURLToPath } from 'url'
 
@@ -112,5 +112,8 @@ export const createMigration: CreateMigration = async function createMigration(
       upSQL: upSQL || `  // Migration code`,
     }),
   )
+
+  writeMigrationIndex({ migrationsDir: payload.db.migrationDir })
+
   payload.logger.info({ msg: `Migration created at ${filePath}.ts` })
 }
diff --git a/packages/db-sqlite/src/index.ts b/packages/db-sqlite/src/index.ts
@@ -93,6 +93,7 @@ export function sqliteAdapter(args: Args): DatabaseAdapterObj<SQLiteAdapter> {
       localesSuffix: args.localesSuffix || '_locales',
       logger: args.logger,
       operators,
+      prodMigrations: args.prodMigrations,
       push: args.push,
       relations: {},
       relationshipsSuffix: args.relationshipsSuffix || '_rels',
diff --git a/packages/db-sqlite/src/types.ts b/packages/db-sqlite/src/types.ts
@@ -18,6 +18,11 @@ export type Args = {
   localesSuffix?: string
   logger?: DrizzleConfig['logger']
   migrationDir?: string
+  prodMigrations?: {
+    down: (args: MigrateDownArgs) => Promise<void>
+    name: string
+    up: (args: MigrateUpArgs) => Promise<void>
+  }[]
   push?: boolean
   relationshipsSuffix?: string
   schemaName?: string
@@ -100,6 +105,11 @@ export type SQLiteAdapter = {
   localesSuffix?: string
   logger: DrizzleConfig['logger']
   operators: Operators
+  prodMigrations?: {
+    down: (args: MigrateDownArgs) => Promise<void>
+    name: string
+    up: (args: MigrateUpArgs) => Promise<void>
+  }[]
   push: boolean
   rejectInitializing: () => void
   relations: Record<string, GenericRelation>
@@ -139,6 +149,11 @@ declare module 'payload' {
     initializing: Promise<void>
     localesSuffix?: string
     logger: DrizzleConfig['logger']
+    prodMigrations?: {
+      down: (args: MigrateDownArgs) => Promise<void>
+      name: string
+      up: (args: MigrateUpArgs) => Promise<void>
+    }[]
     push: boolean
     rejectInitializing: () => void
     relationshipsSuffix?: string
diff --git a/packages/drizzle/src/migrate.ts b/packages/drizzle/src/migrate.ts
@@ -1,4 +1,3 @@
- 
 import type { Payload, PayloadRequest } from 'payload'
 
 import { commitTransaction, initTransaction, killTransaction, readMigrationFiles } from 'payload'
@@ -9,9 +8,12 @@ import type { DrizzleAdapter, Migration } from './types.js'
 import { migrationTableExists } from './utilities/migrationTableExists.js'
 import { parseError } from './utilities/parseError.js'
 
-export async function migrate(this: DrizzleAdapter): Promise<void> {
+export const migrate: DrizzleAdapter['migrate'] = async function migrate(
+  this: DrizzleAdapter,
+  args,
+): Promise<void> {
   const { payload } = this
-  const migrationFiles = await readMigrationFiles({ payload })
+  const migrationFiles = args?.migrations || (await readMigrationFiles({ payload }))
 
   if (!migrationFiles.length) {
     payload.logger.info({ msg: 'No migrations to run.' })
@@ -64,7 +66,7 @@ export async function migrate(this: DrizzleAdapter): Promise<void> {
 
     // If already ran, skip
     if (alreadyRan) {
-      continue  
+      continue
     }
 
     await runMigrationFile(payload, migration, newBatch)
diff --git a/packages/drizzle/src/migrateFresh.ts b/packages/drizzle/src/migrateFresh.ts
@@ -42,7 +42,7 @@ export async function migrateFresh(
 
   await this.dropDatabase({ adapter: this })
 
-  const migrationFiles = (await readMigrationFiles({ payload })) as Migration[]
+  const migrationFiles = await readMigrationFiles({ payload })
   payload.logger.debug({
     msg: `Found ${migrationFiles.length} migration files.`,
   })
diff --git a/packages/drizzle/src/types.ts b/packages/drizzle/src/types.ts
@@ -133,7 +133,7 @@ export type Migration = {
     db?: DrizzleTransaction | LibSQLDatabase<Record<string, never>> | PostgresDB
     payload: Payload
     req: PayloadRequest
-  }) => Promise<boolean>
+  }) => Promise<void>
   up: ({
     db,
     payload,
@@ -142,7 +142,7 @@ export type Migration = {
     db?: DrizzleTransaction | LibSQLDatabase | PostgresDB
     payload: Payload
     req: PayloadRequest
-  }) => Promise<boolean>
+  }) => Promise<void>
 } & MigrationData
 
 export type CreateJSONQueryArgs = {
diff --git a/packages/payload/src/database/migrations/createMigration.ts b/packages/payload/src/database/migrations/createMigration.ts
@@ -2,9 +2,10 @@ import fs from 'fs'
 
 import type { CreateMigration } from '../types.js'
 
+import { writeMigrationIndex } from '../../index.js'
 import { migrationTemplate } from './migrationTemplate.js'
 
-export const createMigration: CreateMigration = async function createMigration({
+export const createMigration: CreateMigration = function createMigration({
   migrationName,
   payload,
 }) {
@@ -23,5 +24,8 @@ export const createMigration: CreateMigration = async function createMigration({
   const fileName = `${timestamp}_${formattedName}.ts`
   const filePath = `${dir}/${fileName}`
   fs.writeFileSync(filePath, migrationTemplate)
+
+  writeMigrationIndex({ migrationsDir: payload.db.migrationDir })
+
   payload.logger.info({ msg: `Migration created at ${filePath}` })
 }
diff --git a/packages/payload/src/database/migrations/migrate.ts b/packages/payload/src/database/migrations/migrate.ts
@@ -7,9 +7,12 @@ import { killTransaction } from '../../utilities/killTransaction.js'
 import { getMigrations } from './getMigrations.js'
 import { readMigrationFiles } from './readMigrationFiles.js'
 
-export async function migrate(this: BaseDatabaseAdapter): Promise<void> {
+export const migrate: BaseDatabaseAdapter['migrate'] = async function migrate(
+  this: BaseDatabaseAdapter,
+  args,
+): Promise<void> {
   const { payload } = this
-  const migrationFiles = await readMigrationFiles({ payload })
+  const migrationFiles = args?.migrations || (await readMigrationFiles({ payload }))
   const { existingMigrations, latestBatch } = await getMigrations({ payload })
 
   const newBatch = latestBatch + 1
diff --git a/packages/payload/src/database/migrations/readMigrationFiles.ts b/packages/payload/src/database/migrations/readMigrationFiles.ts
@@ -27,7 +27,7 @@ export const readMigrationFiles = async ({
     .readdirSync(payload.db.migrationDir)
     .sort()
     .filter((f) => {
-      return f.endsWith('.ts') || f.endsWith('.js')
+      return (f.endsWith('.ts') || f.endsWith('.js')) && !f.includes('index.')
     })
     .map((file) => {
       return path.resolve(payload.db.migrationDir, file)
diff --git a/packages/payload/src/database/migrations/writeMigrationIndex.ts b/packages/payload/src/database/migrations/writeMigrationIndex.ts
@@ -0,0 +1,45 @@
+import fs from 'fs'
+import { getTsconfig } from 'get-tsconfig'
+import path from 'path'
+
+// Function to get all migration files (TS or JS) excluding 'index'
+const getMigrationFiles = (dir: string) => {
+  return fs
+    .readdirSync(dir)
+    .filter(
+      (file) =>
+        (file.endsWith('.ts') || file.endsWith('.js')) &&
+        file !== 'index.ts' &&
+        file !== 'index.js',
+    )
+    .sort()
+}
+
+// Function to generate the index.ts content
+const generateIndexContent = (files: string[]) => {
+  const tsconfig = getTsconfig()
+  const importExt = tsconfig?.config?.compilerOptions?.moduleResolution === 'NodeNext' ? '.js' : ''
+
+  let imports = ''
+  let exportsArray = 'export const migrations = [\n'
+
+  files.forEach((file, index) => {
+    const fileNameWithoutExt = file.replace(/\.[^/.]+$/, '')
+    imports += `import * as migration_${fileNameWithoutExt} from './${fileNameWithoutExt}${importExt}';\n`
+    exportsArray += `  {
+    up: migration_${fileNameWithoutExt}.up,
+    down: migration_${fileNameWithoutExt}.down,
+    name: '${fileNameWithoutExt}'${index !== files.length - 1 ? ',' : ''}\n  },\n`
+  })
+
+  exportsArray += '];\n'
+  return imports + '\n' + exportsArray
+}
+
+// Main function to create the index.ts file
+export const writeMigrationIndex = (args: { migrationsDir: string }) => {
+  const migrationFiles = getMigrationFiles(args.migrationsDir)
+  const indexContent = generateIndexContent(migrationFiles)
+
+  fs.writeFileSync(path.join(args.migrationsDir, 'index.ts'), indexContent)
+}
diff --git a/packages/payload/src/database/types.ts b/packages/payload/src/database/types.ts
diff --git a/packages/payload/src/index.ts b/packages/payload/src/index.ts

Original file line number	Diff line number	Diff line change
`@@ -54,6 +54,10 @@ export const connect: Connect = async function connect(`
`54`	`54`	`this.payload.logger.info('---- DROPPED DATABASE ----')`
`55`	`55`	`}`
`56`	`56`	`}`
	`57`	`+`
	`58`	`+ if (process.env.NODE_ENV === 'production' && this.prodMigrations) {`
	`59`	`+ await this.migrate({ migrations: this.prodMigrations })`
	`60`	`+ }`
`57`	`61`	`} catch (err) {`
`58`	`62`	`console.log(err)`
`59`	`63`	this.payload.logger.error(`Error: cannot connect to MongoDB. Details: ${err.message}`, err)
Original file line number	Diff line number	Diff line change
`@@ -91,4 +91,8 @@ export const connect: Connect = async function connect(`
`91`	`91`	`}`
`92`	`92`
`93`	`93`	`if (typeof this.resolveInitializing === 'function') this.resolveInitializing()`
	`94`	`+`
	`95`	`+ if (process.env.NODE_ENV === 'production' && this.prodMigrations) {`
	`96`	`+ await this.migrate({ migrations: this.prodMigrations })`
	`97`	`+ }`
`94`	`98`	`}`
Original file line number	Diff line number	Diff line change
`@@ -52,4 +52,8 @@ export const connect: Connect = async function connect(`
`52`	`52`	`}`
`53`	`53`
`54`	`54`	`if (typeof this.resolveInitializing === 'function') this.resolveInitializing()`
	`55`	`+`
	`56`	`+ if (process.env.NODE_ENV === 'production' && this.prodMigrations) {`
	`57`	`+ await this.migrate({ migrations: this.prodMigrations })`
	`58`	`+ }`
`55`	`59`	`}`