feat(core/database): add internal migrations

docs/docs/docs/01-core/database/03-migrations.md

@@ -0,0 +1,30 @@
+---
+title: Migrations
+description: Conceptual guide to migrations in Strapi
+tags:
+  - database
+  - migration
+---
+
+Strapi manages schema and data migrations in multiple ways. As much as possible we try to automatically sync the DB schema with the application configuration. However this in not sufficient to manage data migrations or schema migrations that are not reconcilable.
+
+![Migration flowchart](/img/database/migration-flow.png)
+
+## Internal migrations
+
+### Creating a migration
+
+- Add a migration file in `packages/core/database/src/migrations/internal-migrations`
+- Import it in the `index.ts` file of the same folder and add it to the exported array as the last element.
+
+### Migration file format
+
+Every migration should follow this API
+
+```ts
+export default {
+  name: 'name-of-migration',
+  async up(knex: Knex, db: Database): void {},
+  async down(knex: Knex, db: Database): void {},
+};
+```

packages/core/database/src/migrations/__tests__/common.test.ts

@@ -0,0 +1,47 @@
+import { wrapTransaction } from '../common';
+import { Database } from '../..';
+
+describe('wrapTransaction', () => {
+  let db: Database;
+  let fn: jest.Mock;
+  const trx: jest.Mock = jest.fn();
+
+  beforeEach(() => {
+    db = {
+      connection: {
+        transaction: jest.fn().mockImplementation((cb) => cb(trx)),
+      },
+    } as any;
+
+    fn = jest.fn().mockResolvedValue(undefined);
+  });
+
+  it('should wrap the function in a transaction', async () => {
+    const wrappedFn = wrapTransaction(db)(fn);
+    await wrappedFn();
+
+    expect(db.connection.transaction).toHaveBeenCalledWith(expect.any(Function));
+    expect(fn).toHaveBeenCalledWith(trx, db);
+  });
+
+  it('should return the result of the wrapped function', async () => {
+    const result = {};
+    fn.mockResolvedValueOnce(result);
+
+    const wrappedFn = wrapTransaction(db)(fn);
+    const res = await wrappedFn();
+
+    expect(res).toBe(result);
+  });
+
+  it('should rollback the transaction if the wrapped function throws an error', async () => {
+    const error = new Error('Test error');
+    fn.mockRejectedValueOnce(error);
+
+    const wrappedFn = wrapTransaction(db)(fn);
+
+    await expect(wrappedFn()).rejects.toThrow(error);
+    expect(db.connection.transaction).toHaveBeenCalledWith(expect.any(Function));
+    expect(fn).toHaveBeenCalledWith(trx, db);
+  });
+});

packages/core/database/src/migrations/__tests__/storage.test.ts

@@ -0,0 +1,93 @@
+import { createStorage } from '../storage';
+
+import { Database } from '../..';
+
+describe('createStorage', () => {
+  let db: Database;
+  let tableName: string;
+  let storage: ReturnType<typeof createStorage>;
+
+  beforeEach(() => {
+    db = {
+      getSchemaConnection: jest.fn().mockReturnValue({
+        hasTable: jest.fn().mockResolvedValue(false),
+        createTable: jest.fn().mockResolvedValue(undefined),
+      }),
+      getConnection: jest.fn().mockReturnValue({
+        insert: jest.fn().mockReturnValue({
+          into: jest.fn().mockResolvedValue(undefined),
+        }),
+        del: jest.fn().mockReturnValue({
+          where: jest.fn().mockResolvedValue(undefined),
+        }),
+        select: jest.fn().mockReturnValue({
+          from: jest.fn().mockReturnValue({
+            orderBy: jest.fn().mockResolvedValue([]),
+          }),
+        }),
+      }),
+    } as any;
+
+    tableName = 'migrations';
+    storage = createStorage({ db, tableName });
+  });
+
+  describe('logMigration', () => {
+    it('should insert a new migration log', async () => {
+      const name = '20220101120000_create_users_table';
+
+      await storage.logMigration({ name });
+
+      expect(db.getConnection().insert).toHaveBeenCalledWith(
+        expect.objectContaining({ name, time: expect.any(Date) })
+      );
+      expect(db.getConnection().insert({}).into).toHaveBeenCalledWith(tableName);
+    });
+  });
+
+  describe('unlogMigration', () => {
+    it('should delete a migration log', async () => {
+      const name = '20220101120000_create_users_table';
+
+      await storage.unlogMigration({ name });
+
+      expect(db.getConnection().del).toHaveBeenCalled();
+      expect(db.getConnection().del().where).toHaveBeenCalledWith({ name });
+    });
+  });
+
+  describe('executed', () => {
+    it('should create migration table if it does not exist', async () => {
+      await storage.executed();
+
+      expect(db.getSchemaConnection().hasTable).toHaveBeenCalledWith(tableName);
+      expect(db.getSchemaConnection().createTable).toHaveBeenCalledWith(
+        tableName,
+        expect.any(Function)
+      );
+    });
+
+    it('should return an empty array if no migration has been executed', async () => {
+      const result = await storage.executed();
+
+      expect(result).toEqual([]);
+    });
+
+    it('should return an array of executed migration names', async () => {
+      const logs = [
+        { name: '20220101120000_create_users_table' },
+        { name: '20220101130000_create_posts_table' },
+      ];
+
+      (db.getSchemaConnection().hasTable as jest.Mock).mockResolvedValue(true);
+      (db.getConnection().select().from('').orderBy as jest.Mock).mockResolvedValue(logs);
+
+      const result = await storage.executed();
+
+      expect(result).toEqual([
+        '20220101120000_create_users_table',
+        '20220101130000_create_posts_table',
+      ]);
+    });
+  });
+});

packages/core/database/src/migrations/common.ts

@@ -0,0 +1,26 @@
+import type { Resolver } from 'umzug';
+import type { Knex } from 'knex';
+
+import type { Database } from '..';
+
+export interface MigrationProvider {
+  shouldRun(): Promise<boolean>;
+  up(): Promise<void>;
+  down(): Promise<void>;
+}
+
+export type Context = { db: Database };
+
+export type MigrationResolver = Resolver<Context>;
+
+export type MigrationFn = (knex: Knex, db: Database) => Promise<void>;
+
+export type Migration = {
+  name: string;
+  up: MigrationFn;
+  down: MigrationFn;
+};
+
+export const wrapTransaction = (db: Database) => (fn: MigrationFn) => () => {
+  return db.connection.transaction((trx) => Promise.resolve(fn(trx, db)));
+};

packages/core/database/src/migrations/index.ts

@@ -1,93 +1,35 @@
-import path from 'node:path';
-import fse from 'fs-extra';
-import { Umzug } from 'umzug';
-
-import type { Resolver } from 'umzug';
-import type { Knex } from 'knex';
-
-import { createStorage } from './storage';
+import { createUserMigrationProvider } from './users';
+import { createInternalMigrationProvider } from './internal';
 
+import type { MigrationProvider } from './common';
 import type { Database } from '..';
 
-export interface MigrationProvider {
-  shouldRun(): Promise<boolean>;
-  up(): Promise<void>;
-  down(): Promise<void>;
-}
-
-type MigrationResolver = Resolver<{ db: Database }>;
-
-const wrapTransaction = (db: Database) => (fn: (knex: Knex) => unknown) => () => {
-  return db.connection.transaction((trx) => Promise.resolve(fn(trx)));
-};
-
-// TODO: check multiple commands in one sql statement
-const migrationResolver: MigrationResolver = ({ name, path, context }) => {
-  const { db } = context;
-
-  if (!path) {
-    throw new Error(`Migration ${name} has no path`);
-  }
-
-  // if sql file run with knex raw
-  if (path.match(/\.sql$/)) {
-    const sql = fse.readFileSync(path, 'utf8');
-
-    return {
-      name,
-      up: wrapTransaction(db)((knex) => knex.raw(sql)),
-      async down() {
-        throw new Error('Down migration is not supported for sql files');
-      },
-    };
-  }
-
-  // NOTE: we can add some ts register if we want to handle ts migration files at some point
-  // eslint-disable-next-line @typescript-eslint/no-var-requires
-  const migration = require(path);
-  return {
-    name,
-    up: wrapTransaction(db)(migration.up),
-    down: wrapTransaction(db)(migration.down),
-  };
-};
-
-const createUmzugProvider = (db: Database) => {
-  const migrationDir = path.join(strapi.dirs.app.root, 'database/migrations');
-
-  fse.ensureDirSync(migrationDir);
-
-  return new Umzug({
-    storage: createStorage({ db, tableName: 'strapi_migrations' }),
-    logger: console,
-    context: { db },
-    migrations: {
-      glob: ['*.{js,sql}', { cwd: migrationDir }],
-      resolve: migrationResolver,
-    },
-  });
-};
-
-// NOTE: when needed => add internal migrations for core & plugins. How do we overlap them with users migrations ?
+export type { MigrationProvider };
 
-/**
- * Creates migrations provider
- * @type {import('.').createMigrationsProvider}
- */
 export const createMigrationsProvider = (db: Database): MigrationProvider => {
-  const migrations = createUmzugProvider(db);
+  const providers = [createUserMigrationProvider(db), createInternalMigrationProvider(db)];
 
   return {
     async shouldRun() {
-      const pending = await migrations.pending();
+      const shouldRunResponses = await Promise.all(
+        providers.map((provider) => provider.shouldRun())
+      );
 
-      return pending.length > 0 && db.config?.settings?.runMigrations === true;
+      return shouldRunResponses.some((shouldRun) => shouldRun);
     },
     async up() {
-      await migrations.up();
+      for (const provider of providers) {
+        if (await provider.shouldRun()) {
+          await provider.up();
+        }
+      }
     },
     async down() {
-      await migrations.down();
+      for (const provider of providers) {
+        if (await provider.shouldRun()) {
+          await provider.down();
+        }
+      }
     },
   };
 };

packages/core/database/src/migrations/internal-migrations/index.ts

@@ -0,0 +1,12 @@
+import type { Migration } from '../common';
+
+/**
+ * List of all the internal migrations. The array order will be the order in which they are executed.
+ *
+ * {
+ *   name: 'some-name',
+ *   async up(knex: Knex, db: Database) {},
+ *   async down(knex: Knex, db: Database) {},
+ * },
+ */
+export const internalMigrations: Migration[] = [];

packages/core/database/src/migrations/internal.ts

@@ -0,0 +1,38 @@
+import { Umzug } from 'umzug';
+
+import { wrapTransaction } from './common';
+import { internalMigrations } from './internal-migrations';
+import { createStorage } from './storage';
+
+import type { MigrationProvider } from './common';
+import type { Database } from '..';
+
+export const createInternalMigrationProvider = (db: Database): MigrationProvider => {
+  const context = { db };
+
+  const umzugProvider = new Umzug({
+    storage: createStorage({ db, tableName: 'strapi_migrations_internal' }),
+    logger: console,
+    context,
+    migrations: internalMigrations.map((migration) => {
+      return {
+        name: migration.name,
+        up: wrapTransaction(context.db)(migration.up),
+        down: wrapTransaction(context.db)(migration.down),
+      };
+    }),
+  });
+
+  return {
+    async shouldRun() {
+      const pendingMigrations = await umzugProvider.pending();
+      return pendingMigrations.length > 0;
+    },
+    async up() {
+      await umzugProvider.up();
+    },
+    async down() {
+      await umzugProvider.down();
+    },
+  };
+};

packages/core/database/src/migrations/storage.ts

@@ -2,11 +2,11 @@ import type { Database } from '..';
 
 export interface Options {
   db: Database;
-  tableName?: string;
+  tableName: string;
 }
 
 export const createStorage = (opts: Options) => {
-  const { db, tableName = 'strapi_migrations' } = opts;
+  const { db, tableName } = opts;
 
   const hasMigrationTable = () => db.getSchemaConnection().hasTable(tableName);