perf(schema): improve schema inspection speed in SQL drivers (#3549)

.github/workflows/tests.yml

@@ -156,7 +156,7 @@ jobs:
 
       - name: Codecov
         if: steps.changed_packages.outputs.changed_packages != '0'
-        uses: codecov/codecov-action@master
+        uses: codecov/codecov-action@v3
 
       - name: Codeclimate
         if: steps.changed_packages.outputs.changed_packages != '0'

packages/better-sqlite/src/BetterSqliteMikroORM.ts

@@ -13,6 +13,7 @@ export class BetterSqliteMikroORM extends MikroORM<BetterSqliteDriver> {
 
 export type BetterSqliteOptions = Options<BetterSqliteDriver>;
 
+/* istanbul ignore next */
 export function defineBetterSqliteConfig(options: BetterSqliteOptions) {
   return defineConfig({ driver: BetterSqliteDriver, ...options });
 }

packages/better-sqlite/src/BetterSqliteSchemaHelper.ts

@@ -63,7 +63,7 @@ export class BetterSqliteSchemaHelper extends SchemaHelper {
     }, {} as Dictionary<string[]>);
   }
 
-  async getPrimaryKeys(connection: AbstractSqlConnection, indexes: Dictionary, tableName: string, schemaName?: string): Promise<string[]> {
+  async getPrimaryKeys(connection: AbstractSqlConnection, indexes: Index[] = [], tableName: string, schemaName?: string): Promise<string[]> {
     const sql = `pragma table_info(\`${tableName}\`)`;
     const cols = await connection.execute<{ pk: number; name: string }[]>(sql);
 

packages/core/src/platforms/Platform.ts

@@ -263,6 +263,10 @@ export abstract class Platform {
     }
   }
 
+  supportsMultipleStatements(): boolean {
+    return this.config.get('multipleStatements');
+  }
+
   getArrayDeclarationSQL(): string {
     return 'text';
   }

packages/core/src/typings.ts

@@ -491,6 +491,7 @@ export interface EntityMetadata<T = any> {
 }
 
 export interface ISchemaGenerator {
+  /** @deprecated use `dropSchema` and `createSchema` commands respectively */
   generate(): Promise<string>;
   createSchema(options?: { wrap?: boolean; schema?: string }): Promise<void>;
   ensureDatabase(): Promise<boolean>;

packages/core/src/utils/AbstractSchemaGenerator.ts

@@ -23,6 +23,7 @@ export abstract class AbstractSchemaGenerator<D extends IDatabaseDriver> impleme
     this.connection = this.driver.getConnection() as ReturnType<D['getConnection']>;
   }
 
+  /** @deprecated use `dropSchema` and `createSchema` commands respectively */
   async generate(): Promise<string> {
     this.notImplemented();
   }

packages/core/src/utils/Configuration.ts

@@ -42,7 +42,7 @@ import type { Type } from '../types/Type';
 
 export class Configuration<D extends IDatabaseDriver = IDatabaseDriver> {
 
-  static readonly DEFAULTS = {
+  static readonly DEFAULTS: MikroORMOptions = {
     pool: {},
     entities: [],
     entitiesTs: [],
@@ -124,6 +124,7 @@ export class Configuration<D extends IDatabaseDriver = IDatabaseDriver> {
       fileName: (className: string) => className,
     },
     preferReadReplicas: true,
+    /* istanbul ignore next */
     dynamicImportProvider: (id: string) => import(id),
   };
 
@@ -491,6 +492,7 @@ export interface MikroORMOptions<D extends IDatabaseDriver = IDatabaseDriver> ex
   namingStrategy?: { new(): NamingStrategy };
   implicitTransactions?: boolean;
   connect: boolean;
+  verbose: boolean;
   autoJoinOneToOneOwner: boolean;
   propagateToOneOwner: boolean;
   populateAfterFlush: boolean;

packages/core/src/utils/ConfigurationLoader.ts

@@ -250,8 +250,8 @@ export class ConfigurationLoader {
       return;
     }
 
+    /* istanbul ignore next */
     options.dynamicImportProvider ??= id => {
-      /* istanbul ignore next */
       if (platform() === 'win32') {
         try {
           id = fileURLToPath(id);

packages/core/src/utils/Utils.ts

@@ -853,7 +853,7 @@ export class Utils {
    * @see https://github.com/microsoft/TypeScript/issues/43329#issuecomment-922544562
    */
   static async dynamicImport<T = any>(id: string): Promise<T> {
-    if (process.env.TS_JEST || id.endsWith('.json')) {
+    if (id.endsWith('.json') || process.env.TS_JEST) {
       return require(id);
     }
 

packages/knex/src/AbstractSqlDriver.ts

@@ -119,6 +119,7 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
       return this.wrapVirtualExpressionInSubquery(meta, res.getFormattedQuery(), where, options);
     }
 
+    /* istanbul ignore next */
     return res as EntityData<T>[];
   }
 
@@ -142,6 +143,7 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
       return this.wrapVirtualExpressionInSubquery(meta, res.getFormattedQuery(), where, options as Dictionary, QueryType.COUNT);
     }
 
+    /* istanbul ignore next */
     return res as any;
   }
 

packages/knex/src/query/QueryBuilder.ts

@@ -51,23 +51,17 @@ import type { Alias } from './Alias';
 export class QueryBuilder<T extends object = AnyEntity> {
 
   get mainAlias(): Alias {
-    if (!this._mainAlias) {
-      this.throwNoFromClauseError();
-    }
-
-    return this._mainAlias;
+    this.ensureFromClause();
+    return this._mainAlias!;
   }
 
   get alias(): string {
     return this.mainAlias.aliasName;
   }
 
   get helper(): QueryBuilderHelper {
-    if (!this._helper) {
-      this.throwNoFromClauseError();
-    }
-
-    return this._helper;
+    this.ensureFromClause();
+    return this._helper!;
   }
 
   /** @internal */
@@ -1085,8 +1079,11 @@ export class QueryBuilder<T extends object = AnyEntity> {
     return new QueryBuilderHelper(this.mainAlias.entityName, this.mainAlias.aliasName, this._aliases, this.subQueries, this.knex, this.driver);
   }
 
-  private throwNoFromClauseError(): never {
-    throw new Error(`Cannot proceed to build a query because the main alias is not set.`);
+  private ensureFromClause(): void {
+    /* istanbul ignore next */
+    if (!this._mainAlias) {
+      throw new Error(`Cannot proceed to build a query because the main alias is not set.`);
+    }
   }
 
 }

packages/knex/src/schema/DatabaseSchema.ts

@@ -16,9 +16,10 @@ export class DatabaseSchema {
   constructor(private readonly platform: AbstractSqlPlatform,
               readonly name: string) { }
 
-  addTable(name: string, schema: string | undefined | null): DatabaseTable {
+  addTable(name: string, schema: string | undefined | null, comment?: string): DatabaseTable {
     const namespaceName = schema ?? this.name;
     const table = new DatabaseTable(this.platform, name, namespaceName);
+    table.comment = comment;
     this.tables.push(table);
 
     if (namespaceName != null) {
@@ -50,26 +51,12 @@ export class DatabaseSchema {
 
   static async create(connection: AbstractSqlConnection, platform: AbstractSqlPlatform, config: Configuration, schemaName?: string): Promise<DatabaseSchema> {
     const schema = new DatabaseSchema(platform, schemaName ?? config.get('schema'));
-    const tables = await connection.execute<Table[]>(platform.getSchemaHelper()!.getListTablesSQL());
+    const allTables = await connection.execute<Table[]>(platform.getSchemaHelper()!.getListTablesSQL());
     const parts = config.get('migrations').tableName!.split('.');
     const migrationsTableName = parts[1] ?? parts[0];
     const migrationsSchemaName = parts.length > 1 ? parts[0] : config.get('schema', platform.getDefaultSchemaName());
-
-    for (const t of tables) {
-      if (t.table_name === migrationsTableName && (!t.schema_name || t.schema_name === migrationsSchemaName)) {
-        continue;
-      }
-
-      const table = schema.addTable(t.table_name, t.schema_name);
-      table.comment = t.table_comment;
-      const cols = await platform.getSchemaHelper()!.getColumns(connection, table.name, table.schema);
-      const indexes = await platform.getSchemaHelper()!.getIndexes(connection, table.name, table.schema);
-      const checks = await platform.getSchemaHelper()!.getChecks(connection, table.name, table.schema, cols);
-      const pks = await platform.getSchemaHelper()!.getPrimaryKeys(connection, indexes, table.name, table.schema);
-      const fks = await platform.getSchemaHelper()!.getForeignKeys(connection, table.name, table.schema);
-      const enums = await platform.getSchemaHelper()!.getEnumDefinitions(connection, checks, table.name, table.schema);
-      table.init(cols, indexes, checks, pks, fks, enums);
-    }
+    const tables = allTables.filter(t => t.table_name !== migrationsTableName || (t.schema_name && t.schema_name !== migrationsSchemaName));
+    await platform.getSchemaHelper()!.loadInformationSchema(schema, connection, tables);
 
     return schema;
   }

packages/knex/src/schema/DatabaseTable.ts

@@ -39,7 +39,7 @@ export class DatabaseTable {
     return this.checks;
   }
 
-  init(cols: Column[], indexes: Index[], checks: Check[], pks: string[], fks: Dictionary<ForeignKey>, enums: Dictionary<string[]>): void {
+  init(cols: Column[], indexes: Index[] = [], checks: Check[] = [], pks: string[], fks: Dictionary<ForeignKey> = {}, enums: Dictionary<string[]> = {}): void {
     this.indexes = indexes;
     this.checks = checks;
     this.foreignKeys = fks;

packages/knex/src/schema/SchemaGenerator.ts

@@ -1,6 +1,6 @@
 import type { Knex } from 'knex';
 import type { Dictionary, EntityMetadata } from '@mikro-orm/core';
-import { AbstractSchemaGenerator } from '@mikro-orm/core';
+import { AbstractSchemaGenerator, Utils } from '@mikro-orm/core';
 import type { Check, ForeignKey, Index, SchemaDifference, TableDifference } from '../typings';
 import { DatabaseSchema } from './DatabaseSchema';
 import type { DatabaseTable } from './DatabaseTable';
@@ -16,6 +16,7 @@ export class SchemaGenerator extends AbstractSchemaGenerator<AbstractSqlDriver>
   private readonly knex = this.connection.getKnex();
   private readonly options = this.config.get('schemaGenerator');
 
+  /** @deprecated use `dropSchema` and `createSchema` commands respectively */
   async generate(): Promise<string> {
     const [dropSchema, createSchema] = await Promise.all([
       this.getDropSchemaSQL({ wrap: false }),
@@ -442,9 +443,16 @@ export class SchemaGenerator extends AbstractSchemaGenerator<AbstractSqlDriver>
     options.wrap ??= false;
     const lines = this.wrapSchema(sql, options).split('\n').filter(i => i.trim());
 
-    for (const line of lines) {
-      await this.driver.execute(line);
+    if (lines.length === 0) {
+      return;
+    }
+
+    if (this.platform.supportsMultipleStatements()) {
+      const query = lines.join('\n');
+      return void await this.driver.execute(query);
     }
+
+    await Utils.runSerial(lines, line => this.driver.execute(line));
   }
 
   private wrapSchema(sql: string, options: { wrap?: boolean } = {}): string {

packages/knex/src/schema/SchemaHelper.ts

@@ -3,8 +3,9 @@ import type { Connection, Dictionary } from '@mikro-orm/core';
 import { BigIntType, EnumType, Utils } from '@mikro-orm/core';
 import type { AbstractSqlConnection } from '../AbstractSqlConnection';
 import type { AbstractSqlPlatform } from '../AbstractSqlPlatform';
-import type { Check, Column, Index, TableDifference } from '../typings';
+import type { Check, Column, Index, Table, TableDifference } from '../typings';
 import type { DatabaseTable } from './DatabaseTable';
+import type { DatabaseSchema } from './DatabaseSchema';
 
 export abstract class SchemaHelper {
 
@@ -34,20 +35,49 @@ export abstract class SchemaHelper {
     return true;
   }
 
-  async getPrimaryKeys(connection: AbstractSqlConnection, indexes: Index[], tableName: string, schemaName?: string): Promise<string[]> {
-    const pk = indexes.find(i => i.primary);
-    return pk ? pk.columnNames : [];
+  async getPrimaryKeys(connection: AbstractSqlConnection, indexes: Index[] = [], tableName: string, schemaName?: string): Promise<string[]> {
+    const pks = indexes.filter(i => i.primary).map(pk => pk.columnNames);
+    return Utils.flatten(pks);
   }
 
   async getForeignKeys(connection: AbstractSqlConnection, tableName: string, schemaName?: string): Promise<Dictionary> {
     const fks = await connection.execute<any[]>(this.getForeignKeysSQL(tableName, schemaName));
     return this.mapForeignKeys(fks, tableName, schemaName);
   }
 
+  protected getTableKey(t: Table) {
+    const unquote = (str: string) => str.replace(/['"`]/g, '');
+    const parts = t.table_name.split('.');
+
+    if (parts.length > 1) {
+      return `${unquote(parts[0])}.${unquote(parts[1])}`;
+    }
+
+    if (t.schema_name) {
+      return `${unquote(t.schema_name)}.${unquote(t.table_name)}`;
+    }
+
+    return unquote(t.table_name);
+  }
+
   async getEnumDefinitions(connection: AbstractSqlConnection, checks: Check[], tableName: string, schemaName?: string): Promise<Dictionary<string[]>> {
     return {};
   }
 
+  async loadInformationSchema(schema: DatabaseSchema, connection: AbstractSqlConnection, tables: Table[]): Promise<void> {
+    for (const t of tables) {
+      const table = schema.addTable(t.table_name, t.schema_name);
+      table.comment = t.table_comment;
+      const cols = await this.getColumns(connection, table.name, table.schema);
+      const indexes = await this.getIndexes(connection, table.name, table.schema);
+      const checks = await this.getChecks(connection, table.name, table.schema, cols);
+      const pks = await this.getPrimaryKeys(connection, indexes, table.name, table.schema);
+      const fks = await this.getForeignKeys(connection, table.name, table.schema);
+      const enums = await this.getEnumDefinitions(connection, checks, table.name, table.schema);
+      table.init(cols, indexes, checks, pks, fks, enums);
+    }
+  }
+
   getListTablesSQL(schemaName?: string): string {
     throw new Error('Not supported by given driver');
   }

packages/mariadb/src/MariaDbMikroORM.ts

@@ -13,6 +13,7 @@ export class MariaDbMikroORM extends MikroORM<MariaDbDriver> {
 
 export type MariaDbOptions = Options<MariaDbDriver>;
 
+/* istanbul ignore next */
 export function defineMariaDbConfig(options: MariaDbOptions) {
   return defineConfig({ driver: MariaDbDriver, ...options });
 }