-
-
Notifications
You must be signed in to change notification settings - Fork 7.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #18437 from strapi/v5/internal-migration
- Loading branch information
Showing
11 changed files
with
343 additions
and
80 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
--- | ||
title: Migrations | ||
description: Conceptual guide to migrations in Strapi | ||
tags: | ||
- database | ||
- migration | ||
--- | ||
|
||
Strapi manages schema and data migrations in multiple ways. As much as possible we try to automatically sync the DB schema with the application configuration. However this in not sufficient to manage data migrations or schema migrations that are not reconcilable. | ||
|
||
![Migration flowchart](/img/database/migration-flow.png) | ||
|
||
## Internal migrations | ||
|
||
### Creating a migration | ||
|
||
- Add a migration file in `packages/core/database/src/migrations/internal-migrations` | ||
- Import it in the `index.ts` file of the same folder and add it to the exported array as the last element. | ||
|
||
### Migration file format | ||
|
||
Every migration should follow this API | ||
|
||
```ts | ||
export default { | ||
name: 'name-of-migration', | ||
async up(knex: Knex, db: Database): void {}, | ||
async down(knex: Knex, db: Database): void {}, | ||
}; | ||
``` |
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
47 changes: 47 additions & 0 deletions
47
packages/core/database/src/migrations/__tests__/common.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
import { wrapTransaction } from '../common'; | ||
import { Database } from '../..'; | ||
|
||
describe('wrapTransaction', () => { | ||
let db: Database; | ||
let fn: jest.Mock; | ||
const trx: jest.Mock = jest.fn(); | ||
|
||
beforeEach(() => { | ||
db = { | ||
connection: { | ||
transaction: jest.fn().mockImplementation((cb) => cb(trx)), | ||
}, | ||
} as any; | ||
|
||
fn = jest.fn().mockResolvedValue(undefined); | ||
}); | ||
|
||
it('should wrap the function in a transaction', async () => { | ||
const wrappedFn = wrapTransaction(db)(fn); | ||
await wrappedFn(); | ||
|
||
expect(db.connection.transaction).toHaveBeenCalledWith(expect.any(Function)); | ||
expect(fn).toHaveBeenCalledWith(trx, db); | ||
}); | ||
|
||
it('should return the result of the wrapped function', async () => { | ||
const result = {}; | ||
fn.mockResolvedValueOnce(result); | ||
|
||
const wrappedFn = wrapTransaction(db)(fn); | ||
const res = await wrappedFn(); | ||
|
||
expect(res).toBe(result); | ||
}); | ||
|
||
it('should rollback the transaction if the wrapped function throws an error', async () => { | ||
const error = new Error('Test error'); | ||
fn.mockRejectedValueOnce(error); | ||
|
||
const wrappedFn = wrapTransaction(db)(fn); | ||
|
||
await expect(wrappedFn()).rejects.toThrow(error); | ||
expect(db.connection.transaction).toHaveBeenCalledWith(expect.any(Function)); | ||
expect(fn).toHaveBeenCalledWith(trx, db); | ||
}); | ||
}); |
93 changes: 93 additions & 0 deletions
93
packages/core/database/src/migrations/__tests__/storage.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,93 @@ | ||
import { createStorage } from '../storage'; | ||
|
||
import { Database } from '../..'; | ||
|
||
describe('createStorage', () => { | ||
let db: Database; | ||
let tableName: string; | ||
let storage: ReturnType<typeof createStorage>; | ||
|
||
beforeEach(() => { | ||
db = { | ||
getSchemaConnection: jest.fn().mockReturnValue({ | ||
hasTable: jest.fn().mockResolvedValue(false), | ||
createTable: jest.fn().mockResolvedValue(undefined), | ||
}), | ||
getConnection: jest.fn().mockReturnValue({ | ||
insert: jest.fn().mockReturnValue({ | ||
into: jest.fn().mockResolvedValue(undefined), | ||
}), | ||
del: jest.fn().mockReturnValue({ | ||
where: jest.fn().mockResolvedValue(undefined), | ||
}), | ||
select: jest.fn().mockReturnValue({ | ||
from: jest.fn().mockReturnValue({ | ||
orderBy: jest.fn().mockResolvedValue([]), | ||
}), | ||
}), | ||
}), | ||
} as any; | ||
|
||
tableName = 'migrations'; | ||
storage = createStorage({ db, tableName }); | ||
}); | ||
|
||
describe('logMigration', () => { | ||
it('should insert a new migration log', async () => { | ||
const name = '20220101120000_create_users_table'; | ||
|
||
await storage.logMigration({ name }); | ||
|
||
expect(db.getConnection().insert).toHaveBeenCalledWith( | ||
expect.objectContaining({ name, time: expect.any(Date) }) | ||
); | ||
expect(db.getConnection().insert({}).into).toHaveBeenCalledWith(tableName); | ||
}); | ||
}); | ||
|
||
describe('unlogMigration', () => { | ||
it('should delete a migration log', async () => { | ||
const name = '20220101120000_create_users_table'; | ||
|
||
await storage.unlogMigration({ name }); | ||
|
||
expect(db.getConnection().del).toHaveBeenCalled(); | ||
expect(db.getConnection().del().where).toHaveBeenCalledWith({ name }); | ||
}); | ||
}); | ||
|
||
describe('executed', () => { | ||
it('should create migration table if it does not exist', async () => { | ||
await storage.executed(); | ||
|
||
expect(db.getSchemaConnection().hasTable).toHaveBeenCalledWith(tableName); | ||
expect(db.getSchemaConnection().createTable).toHaveBeenCalledWith( | ||
tableName, | ||
expect.any(Function) | ||
); | ||
}); | ||
|
||
it('should return an empty array if no migration has been executed', async () => { | ||
const result = await storage.executed(); | ||
|
||
expect(result).toEqual([]); | ||
}); | ||
|
||
it('should return an array of executed migration names', async () => { | ||
const logs = [ | ||
{ name: '20220101120000_create_users_table' }, | ||
{ name: '20220101130000_create_posts_table' }, | ||
]; | ||
|
||
(db.getSchemaConnection().hasTable as jest.Mock).mockResolvedValue(true); | ||
(db.getConnection().select().from('').orderBy as jest.Mock).mockResolvedValue(logs); | ||
|
||
const result = await storage.executed(); | ||
|
||
expect(result).toEqual([ | ||
'20220101120000_create_users_table', | ||
'20220101130000_create_posts_table', | ||
]); | ||
}); | ||
}); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
import type { Resolver } from 'umzug'; | ||
import type { Knex } from 'knex'; | ||
|
||
import type { Database } from '..'; | ||
|
||
export interface MigrationProvider { | ||
shouldRun(): Promise<boolean>; | ||
up(): Promise<void>; | ||
down(): Promise<void>; | ||
} | ||
|
||
export type Context = { db: Database }; | ||
|
||
export type MigrationResolver = Resolver<Context>; | ||
|
||
export type MigrationFn = (knex: Knex, db: Database) => Promise<void>; | ||
|
||
export type Migration = { | ||
name: string; | ||
up: MigrationFn; | ||
down: MigrationFn; | ||
}; | ||
|
||
export const wrapTransaction = (db: Database) => (fn: MigrationFn) => () => { | ||
return db.connection.transaction((trx) => Promise.resolve(fn(trx, db))); | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,93 +1,35 @@ | ||
import path from 'node:path'; | ||
import fse from 'fs-extra'; | ||
import { Umzug } from 'umzug'; | ||
|
||
import type { Resolver } from 'umzug'; | ||
import type { Knex } from 'knex'; | ||
|
||
import { createStorage } from './storage'; | ||
import { createUserMigrationProvider } from './users'; | ||
import { createInternalMigrationProvider } from './internal'; | ||
|
||
import type { MigrationProvider } from './common'; | ||
import type { Database } from '..'; | ||
|
||
export interface MigrationProvider { | ||
shouldRun(): Promise<boolean>; | ||
up(): Promise<void>; | ||
down(): Promise<void>; | ||
} | ||
|
||
type MigrationResolver = Resolver<{ db: Database }>; | ||
|
||
const wrapTransaction = (db: Database) => (fn: (knex: Knex) => unknown) => () => { | ||
return db.connection.transaction((trx) => Promise.resolve(fn(trx))); | ||
}; | ||
|
||
// TODO: check multiple commands in one sql statement | ||
const migrationResolver: MigrationResolver = ({ name, path, context }) => { | ||
const { db } = context; | ||
|
||
if (!path) { | ||
throw new Error(`Migration ${name} has no path`); | ||
} | ||
|
||
// if sql file run with knex raw | ||
if (path.match(/\.sql$/)) { | ||
const sql = fse.readFileSync(path, 'utf8'); | ||
|
||
return { | ||
name, | ||
up: wrapTransaction(db)((knex) => knex.raw(sql)), | ||
async down() { | ||
throw new Error('Down migration is not supported for sql files'); | ||
}, | ||
}; | ||
} | ||
|
||
// NOTE: we can add some ts register if we want to handle ts migration files at some point | ||
// eslint-disable-next-line @typescript-eslint/no-var-requires | ||
const migration = require(path); | ||
return { | ||
name, | ||
up: wrapTransaction(db)(migration.up), | ||
down: wrapTransaction(db)(migration.down), | ||
}; | ||
}; | ||
|
||
const createUmzugProvider = (db: Database) => { | ||
const migrationDir = path.join(strapi.dirs.app.root, 'database/migrations'); | ||
|
||
fse.ensureDirSync(migrationDir); | ||
|
||
return new Umzug({ | ||
storage: createStorage({ db, tableName: 'strapi_migrations' }), | ||
logger: console, | ||
context: { db }, | ||
migrations: { | ||
glob: ['*.{js,sql}', { cwd: migrationDir }], | ||
resolve: migrationResolver, | ||
}, | ||
}); | ||
}; | ||
|
||
// NOTE: when needed => add internal migrations for core & plugins. How do we overlap them with users migrations ? | ||
export type { MigrationProvider }; | ||
|
||
/** | ||
* Creates migrations provider | ||
* @type {import('.').createMigrationsProvider} | ||
*/ | ||
export const createMigrationsProvider = (db: Database): MigrationProvider => { | ||
const migrations = createUmzugProvider(db); | ||
const providers = [createUserMigrationProvider(db), createInternalMigrationProvider(db)]; | ||
|
||
return { | ||
async shouldRun() { | ||
const pending = await migrations.pending(); | ||
const shouldRunResponses = await Promise.all( | ||
providers.map((provider) => provider.shouldRun()) | ||
); | ||
|
||
return pending.length > 0 && db.config?.settings?.runMigrations === true; | ||
return shouldRunResponses.some((shouldRun) => shouldRun); | ||
}, | ||
async up() { | ||
await migrations.up(); | ||
for (const provider of providers) { | ||
if (await provider.shouldRun()) { | ||
await provider.up(); | ||
} | ||
} | ||
}, | ||
async down() { | ||
await migrations.down(); | ||
for (const provider of providers) { | ||
if (await provider.shouldRun()) { | ||
await provider.down(); | ||
} | ||
} | ||
}, | ||
}; | ||
}; |
12 changes: 12 additions & 0 deletions
12
packages/core/database/src/migrations/internal-migrations/index.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
import type { Migration } from '../common'; | ||
|
||
/** | ||
* List of all the internal migrations. The array order will be the order in which they are executed. | ||
* | ||
* { | ||
* name: 'some-name', | ||
* async up(knex: Knex, db: Database) {}, | ||
* async down(knex: Knex, db: Database) {}, | ||
* }, | ||
*/ | ||
export const internalMigrations: Migration[] = []; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
import { Umzug } from 'umzug'; | ||
|
||
import { wrapTransaction } from './common'; | ||
import { internalMigrations } from './internal-migrations'; | ||
import { createStorage } from './storage'; | ||
|
||
import type { MigrationProvider } from './common'; | ||
import type { Database } from '..'; | ||
|
||
export const createInternalMigrationProvider = (db: Database): MigrationProvider => { | ||
const context = { db }; | ||
|
||
const umzugProvider = new Umzug({ | ||
storage: createStorage({ db, tableName: 'strapi_migrations_internal' }), | ||
logger: console, | ||
context, | ||
migrations: internalMigrations.map((migration) => { | ||
return { | ||
name: migration.name, | ||
up: wrapTransaction(context.db)(migration.up), | ||
down: wrapTransaction(context.db)(migration.down), | ||
}; | ||
}), | ||
}); | ||
|
||
return { | ||
async shouldRun() { | ||
const pendingMigrations = await umzugProvider.pending(); | ||
return pendingMigrations.length > 0; | ||
}, | ||
async up() { | ||
await umzugProvider.up(); | ||
}, | ||
async down() { | ||
await umzugProvider.down(); | ||
}, | ||
}; | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.