perf(core): interpolate query parameters at ORM level

To get around limitations on query parameter limits, as well as to make it faster, we now interpolate all query params manually, passing raw query to the connector. This also allows to print the actual queries in the debug log. Previously output of knex was used, but that was just an estimation, not a real query (e.g. dates were presented as ISO strings, but stored as numbers/timestamps). + Mapping of DB results now also uses JIT compilation. Related: #732
mikro-orm · Oct 7, 2020 · 742b813 · 742b813
1 parent 0a2299f
commit 742b813
Show file tree

Hide file tree

Showing 37 changed files with 383 additions and 144 deletions.
diff --git a/package.json b/package.json
@@ -109,6 +109,7 @@
     "@types/mysql2": "types/mysql2#d4ef3b2292f328049f7e4c545f6adab7d6a350a9",
     "@types/node": "^14.6.2",
     "@types/pg": "^7.14.4",
+    "@types/sqlstring": "^2.2.1",
     "@types/uuid": "^8.3.0",
     "@types/webpack-env": "^1.15.2",
     "@typescript-eslint/eslint-plugin": "~3.10.0",

diff --git a/packages/core/src/connections/Connection.ts b/packages/core/src/connections/Connection.ts
@@ -4,10 +4,12 @@ import c from 'ansi-colors';
 import { Configuration, ConnectionOptions, Utils } from '../utils';
 import { MetadataStorage } from '../metadata';
 import { Dictionary } from '../typings';
+import { Platform } from '../platforms/Platform';
 
 export abstract class Connection {
 
   protected metadata!: MetadataStorage;
+  protected platform!: Platform;
   protected abstract client: any;
 
   constructor(protected readonly config: Configuration,
@@ -80,6 +82,10 @@ export abstract class Connection {
     this.metadata = metadata;
   }
 
+  setPlatform(platform: Platform): void {
+    this.platform = platform;
+  }
+
   protected async executeQuery<T>(query: string, cb: () => Promise<T>): Promise<T> {
     const now = Date.now();
 

diff --git a/packages/core/src/drivers/DatabaseDriver.ts b/packages/core/src/drivers/DatabaseDriver.ts
@@ -2,7 +2,7 @@ import { CountOptions, EntityManagerType, FindOneOptions, FindOptions, IDatabase
 import { EntityData, EntityMetadata, EntityProperty, FilterQuery, AnyEntity, Dictionary, Primary, PopulateOptions } from '../typings';
 import { MetadataStorage } from '../metadata';
 import { Connection, QueryResult, Transaction } from '../connections';
-import { Configuration, ConnectionOptions, Utils } from '../utils';
+import { Configuration, ConnectionOptions, EntityComparator, Utils } from '../utils';
 import { LockMode, QueryOrder, QueryOrderMap, ReferenceType } from '../enums';
 import { Platform } from '../platforms';
 import { Collection } from '../entity';
@@ -18,6 +18,7 @@ export abstract class DatabaseDriver<C extends Connection> implements IDatabaseD
   protected readonly replicas: C[] = [];
   protected readonly platform!: Platform;
   protected readonly logger = this.config.getLogger();
+  protected comparator!: EntityComparator;
   protected metadata!: MetadataStorage;
 
   protected constructor(protected readonly config: Configuration,
@@ -59,32 +60,12 @@ export abstract class DatabaseDriver<C extends Connection> implements IDatabaseD
     await this.nativeUpdate<T>(coll.owner.constructor.name, coll.owner.__helper!.__primaryKey, data, ctx);
   }
 
-  mapResult<T extends AnyEntity<T>>(result: EntityData<T>, meta: EntityMetadata, populate: PopulateOptions<T>[] = []): EntityData<T> | null {
+  mapResult<T extends AnyEntity<T>>(result: EntityData<T>, meta: EntityMetadata<T>, populate: PopulateOptions<T>[] = []): EntityData<T> | null {
     if (!result || !meta) {
       return null;
     }
 
-    const ret = Object.assign({}, result) as any;
-
-    meta.props.forEach(prop => {
-      if (prop.fieldNames && prop.fieldNames.length > 1 && prop.fieldNames.every(joinColumn => Utils.isDefined(ret[joinColumn], true))) {
-        const temp: any[] = [];
-        prop.fieldNames.forEach(joinColumn => {
-          temp.push(ret[joinColumn]);
-          delete ret[joinColumn];
-        });
-
-        ret[prop.name] = temp;
-      } else if (prop.fieldNames && prop.fieldNames[0] in ret) {
-        Utils.renameKey(ret, prop.fieldNames[0], prop.name);
-      }
-
-      if (prop.type === 'boolean' && ![null, undefined].includes(ret[prop.name])) {
-        ret[prop.name] = !!ret[prop.name];
-      }
-    });
-
-    return ret;
+    return this.comparator.mapResult(meta.className, result);
   }
 
   async connect(): Promise<C> {
@@ -120,7 +101,13 @@ export abstract class DatabaseDriver<C extends Connection> implements IDatabaseD
 
   setMetadata(metadata: MetadataStorage): void {
     this.metadata = metadata;
+    this.comparator = new EntityComparator(this.metadata, this.platform);
     this.connection.setMetadata(metadata);
+    this.connection.setPlatform(this.platform);
+    this.replicas.forEach(replica => {
+      replica.setMetadata(metadata);
+      replica.setPlatform(this.platform);
+    });
   }
 
   getDependencies(): string[] {

diff --git a/packages/core/src/entity/WrappedEntity.ts b/packages/core/src/entity/WrappedEntity.ts
@@ -1,3 +1,4 @@
+import { inspect } from 'util';
 import { EntityManager } from '../EntityManager';
 import { AnyEntity, Dictionary, EntityData, EntityMetadata, Populate, Primary } from '../typings';
 import { IdentifiedReference, Reference } from './Reference';
@@ -6,7 +7,6 @@ import { AssignOptions, EntityAssigner } from './EntityAssigner';
 import { Utils } from '../utils/Utils';
 import { LockMode } from '../enums';
 import { ValidationError } from '../errors';
-import { Platform } from '../platforms/Platform';
 
 export class WrappedEntity<T extends AnyEntity<T>, PK extends keyof T> {
 
@@ -67,7 +67,7 @@ export class WrappedEntity<T extends AnyEntity<T>, PK extends keyof T> {
   }
 
   hasPrimaryKey(): boolean {
-    return this.__meta.primaryKeys.every(pk => {
+    return this.entity.__meta!.primaryKeys.every(pk => {
       const val = Utils.extractPK(this.entity[pk]);
       return val !== undefined && val !== null;
     });
@@ -77,36 +77,40 @@ export class WrappedEntity<T extends AnyEntity<T>, PK extends keyof T> {
     return this.entity.__meta!;
   }
 
-  get __platform(): Platform {
+  get __platform() {
     return this.entity.__platform!;
   }
 
   get __primaryKey(): Primary<T> {
-    return Utils.getPrimaryKeyValue(this.entity, this.__meta.primaryKeys);
+    return Utils.getPrimaryKeyValue(this.entity, this.entity.__meta!.primaryKeys);
   }
 
   set __primaryKey(id: Primary<T>) {
-    this.entity[this.__meta.primaryKeys[0] as string] = id;
+    this.entity[this.entity.__meta!.primaryKeys[0] as string] = id;
   }
 
   get __primaryKeys(): Primary<T>[] {
-    return Utils.getPrimaryKeyValues(this.entity, this.__meta.primaryKeys);
+    return Utils.getPrimaryKeyValues(this.entity, this.entity.__meta!.primaryKeys);
   }
 
   get __primaryKeyCond(): Primary<T> | Primary<T>[] {
-    if (this.__meta.compositePK) {
+    if (this.entity.__meta!.compositePK) {
       return this.__primaryKeys;
     }
 
     return this.__primaryKey;
   }
 
   get __serializedPrimaryKey(): Primary<T> | string {
-    if (this.__meta.compositePK) {
-      return Utils.getCompositeKeyHash(this.entity, this.__meta);
+    if (this.entity.__meta!.simplePK) {
+      return '' + this.entity[this.entity.__meta!.serializedPrimaryKey];
     }
 
-    const value = this.entity[this.__meta.serializedPrimaryKey];
+    if (this.entity.__meta!.compositePK) {
+      return Utils.getCompositeKeyHash(this.entity, this.entity.__meta!);
+    }
+
+    const value = this.entity[this.entity.__meta!.serializedPrimaryKey];
 
     if (Utils.isEntity<T>(value)) {
       return value.__helper!.__serializedPrimaryKey;
@@ -115,4 +119,8 @@ export class WrappedEntity<T extends AnyEntity<T>, PK extends keyof T> {
     return '' + value;
   }
 
+  [inspect.custom]() {
+    return `[WrappedEntity<${this.entity.__meta!.className}>]`;
+  }
+
 }
diff --git a/packages/core/src/metadata/MetadataDiscovery.ts b/packages/core/src/metadata/MetadataDiscovery.ts
@@ -11,7 +11,7 @@ import { EntitySchema } from './EntitySchema';
 import { Cascade, ReferenceType } from '../enums';
 import { MetadataError } from '../errors';
 import { Platform } from '../platforms';
-import { ArrayType, BlobType, Type } from '../types';
+import { ArrayType, BlobType, JsonType, Type } from '../types';
 import { EntityComparator } from '../utils/EntityComparator';
 
 export class MetadataDiscovery {
@@ -52,6 +52,8 @@ export class MetadataDiscovery {
     this.discovered.forEach(meta => {
       const root = Utils.getRootEntity(this.metadata, meta);
       const props = Object.values(meta.properties);
+      const pk = meta.properties[meta.primaryKeys[0]];
+      meta.simplePK = !meta.compositePK && pk?.reference === ReferenceType.SCALAR && !pk.customType;
       meta.props = [...props.filter(p => p.primary), ...props.filter(p => !p.primary)];
       meta.relations = meta.props.filter(prop => prop.reference !== ReferenceType.SCALAR && prop.reference !== ReferenceType.EMBEDDED);
       meta.comparableProps = meta.props.filter(prop => EntityComparator.isComparable(prop, root));
@@ -664,6 +666,10 @@ export class MetadataDiscovery {
       prop.customType = new BlobType();
     }
 
+    if (!prop.customType && !prop.columnTypes && prop.type === 'json') {
+      prop.customType = new JsonType();
+    }
+
     if (prop.type as unknown instanceof Type) {
       prop.customType = prop.type as unknown as Type<any>;
     }

diff --git a/packages/core/src/platforms/Platform.ts b/packages/core/src/platforms/Platform.ts
@@ -3,10 +3,13 @@ import { NamingStrategy, UnderscoreNamingStrategy } from '../naming-strategy';
 import { Constructor, Dictionary, EntityProperty, IPrimaryKey, Primary, ISchemaGenerator } from '../typings';
 import { ExceptionConverter } from './ExceptionConverter';
 import { EntityManager } from '../EntityManager';
+import { Configuration } from '../utils/Configuration';
 
 export abstract class Platform {
 
   protected readonly exceptionConverter = new ExceptionConverter();
+  protected config!: Configuration;
+  protected timezone?: string;
 
   usesPivotTable(): boolean {
     return false;
@@ -158,4 +161,14 @@ export abstract class Platform {
     return value as string;
   }
 
+  setConfig(config: Configuration): void {
+    this.config = config;
+
+    if (this.config.get('forceUtcTimezone')) {
+      this.timezone = 'Z';
+    } else {
+      this.timezone = this.config.get('timezone');
+    }
+  }
+
 }
diff --git a/packages/core/src/typings.ts b/packages/core/src/typings.ts
@@ -185,6 +185,7 @@ export interface EntityMetadata<T extends AnyEntity<T> = any> {
   path: string;
   primaryKeys: (keyof T & string)[];
   compositePK: boolean;
+  simplePK: boolean; // PK is scalar, no custom types or composite keys
   versionProperty: keyof T & string;
   serializedPrimaryKey: keyof T & string;
   properties: { [K in keyof T & string]: EntityProperty<T> };

diff --git a/packages/core/src/unit-of-work/ChangeSetPersister.ts b/packages/core/src/unit-of-work/ChangeSetPersister.ts
@@ -85,17 +85,16 @@ export class ChangeSetPersister {
     for (let i = 0; i < changeSets.length; i += size) {
       const chunk = changeSets.slice(i, i + size);
       await this.persistNewEntitiesBatch(meta, chunk, ctx);
-      await this.reloadVersionValues(meta, chunk, ctx);
+
+      if (!this.platform.usesReturningStatement()) {
+        await this.reloadVersionValues(meta, chunk, ctx);
+      }
     }
   }
 
   private async persistNewEntitiesBatch<T extends AnyEntity<T>>(meta: EntityMetadata<T>, changeSets: ChangeSet<T>[], ctx?: Transaction): Promise<void> {
     const res = await this.driver.nativeInsertMany(meta.className, changeSets.map(cs => cs.payload), ctx);
 
-    if (!this.platform.usesReturningStatement()) {
-      await this.reloadVersionValues(meta, changeSets, ctx);
-    }
-
     for (let i = 0; i < changeSets.length; i++) {
       const changeSet = changeSets[i];
       const wrapped = changeSet.entity.__helper!;
@@ -105,7 +104,6 @@ export class ChangeSetPersister {
       }
 
       this.mapReturnedValues(changeSet, res, meta);
-
       this.markAsPopulated(changeSet, meta);
       wrapped.__initialized = true;
       wrapped.__managed = true;

diff --git a/packages/core/src/unit-of-work/UnitOfWork.ts b/packages/core/src/unit-of-work/UnitOfWork.ts
@@ -314,7 +314,6 @@ export class UnitOfWork {
       this.checkOrphanRemoval(changeSet);
       this.changeSets.set(entity, changeSet);
       this.persistStack.delete(entity);
-      wrapped.__originalEntityData = changeSet.payload;
     }
   }
 

diff --git a/packages/core/src/utils/Configuration.ts b/packages/core/src/utils/Configuration.ts
@@ -49,7 +49,7 @@ export class Configuration<D extends IDatabaseDriver = IDatabaseDriver> {
     populateAfterFlush: false,
     forceUtcTimezone: false,
     ensureIndexes: false,
-    batchSize: 1000,
+    batchSize: 300,
     debug: false,
     verbose: false,
     driverOptions: {},
@@ -99,6 +99,7 @@ export class Configuration<D extends IDatabaseDriver = IDatabaseDriver> {
     this.logger = new Logger(this.options.logger, this.options.debug);
     this.driver = this.initDriver();
     this.platform = this.driver.getPlatform();
+    this.platform.setConfig(this);
     this.init();
   }
 

diff --git a/packages/core/src/utils/EntityComparator.ts b/packages/core/src/utils/EntityComparator.ts
@@ -5,11 +5,13 @@ import { Platform } from '../platforms';
 import { Utils, compareArrays, compareBuffers, compareObjects, equals } from './Utils';
 
 type Comparator<T> = (a: T, b: T) => EntityData<T>;
+type ResultMapper<T> = (result: EntityData<T>) => EntityData<T> | null;
 type SnapshotGenerator<T> = (entity: T) => EntityData<T>;
 
 export class EntityComparator {
 
   private readonly comparators = new Map<string, Comparator<any>>();
+  private readonly mappers = new Map<string, ResultMapper<any>>();
   private readonly snapshotGenerators = new Map<string, SnapshotGenerator<any>>();
 
   constructor(private readonly metadata: IMetadataStorage,
@@ -32,6 +34,14 @@ export class EntityComparator {
     return generator(entity);
   }
 
+  /**
+   * Maps database columns to properties.
+   */
+  mapResult<T extends AnyEntity<T>>(entityName: string, result: EntityData<T>): EntityData<T> | null {
+    const mapper = this.getResultMapper<T>(entityName);
+    return mapper(result);
+  }
+
   /**
    * @internal Highly performance-sensitive method.
    */
@@ -63,6 +73,51 @@ export class EntityComparator {
     return snapshotGenerator;
   }
 
+  /**
+   * @internal Highly performance-sensitive method.
+   */
+  getResultMapper<T extends AnyEntity<T>>(entityName: string): ResultMapper<T> {
+    const exists = this.mappers.get(entityName);
+
+    if (exists) {
+      return exists;
+    }
+
+    const meta = this.metadata.find<T>(entityName)!;
+
+    if (!meta) {
+      return i => i;
+    }
+
+    const props: string[] = [];
+    const context = new Map<string, any>();
+
+    props.push(`  const mapped = {};`);
+    meta.props.forEach(prop => {
+      if (prop.fieldNames) {
+        if (prop.fieldNames.length > 1) {
+          props.push(`  if (${prop.fieldNames.map(field => `result.${field} != null`).join(' && ')}) {\n    ret.${prop.name} = [${prop.fieldNames.map(field => `result.${field}`).join(', ')}];`);
+          props.push(...prop.fieldNames.map(field => `    mapped.${field} = true;`));
+          props.push(`  } else if (${prop.fieldNames.map(field => `result.${field} == null`).join(' && ')}) {\n    ret.${prop.name} = null;`);
+          props.push(...prop.fieldNames.map(field => `    mapped.${field} = true;`), '  }');
+        } else {
+          if (prop.type === 'boolean') {
+            props.push(`  if ('${prop.fieldNames[0]}' in result) { ret.${prop.name} = result.${prop.fieldNames[0]} == null ? result.${prop.fieldNames[0]} : !!result.${prop.fieldNames[0]}; mapped.${prop.fieldNames[0]} = true; }`);
+          } else {
+            props.push(`  if ('${prop.fieldNames[0]}' in result) { ret.${prop.name} = result.${prop.fieldNames[0]}; mapped.${prop.fieldNames[0]} = true; }`);
+          }
+        }
+      }
+    });
+    props.push(`  for (let k in result) { if (result.hasOwnProperty(k) && !mapped[k]) ret[k] = result[k]; }`);
+
+    const code = `return function(result) {\n  const ret = {};\n${props.join('\n')}\n  return ret;\n}`;
+    const snapshotGenerator = this.createFunction(context, code);
+    this.mappers.set(entityName, snapshotGenerator);
+
+    return snapshotGenerator;
+  }
+
   /* istanbul ignore next */
   private createFunction(context: Map<string, any>, code: string) {
     try {