feat(core): add support for alternative loading strategies (#556)

This adds support for alternate load strategies. As of now, only one additional strategy is supported: `joined`. Currently, the only way to specify the strategy is on on the decorator for the relationship:

```ts
@entity()
export class Author2 {
  @OneToMany({
    entity: () => Book2,
    mappedBy: 'author',
    orderBy: { title: QueryOrder.ASC },
    strategy: LoadStrategy.JOINED, // new property
  })
  books!: Collection<Book2>;
}
```

Related: #440

packages/core/src/EntityManager.ts

@@ -3,7 +3,7 @@ import { v4 as uuid } from 'uuid';
 import { Configuration, RequestContext, Utils, ValidationError, SmartQueryHelper } from './utils';
 import { EntityAssigner, EntityFactory, EntityLoader, EntityRepository, EntityValidator, IdentifiedReference, Reference, ReferenceType, wrap } from './entity';
 import { LockMode, UnitOfWork } from './unit-of-work';
-import { IDatabaseDriver, FindOneOptions, FindOptions, EntityManagerType } from './drivers';
+import { IDatabaseDriver, FindOneOptions, FindOptions, EntityManagerType, Populate, PopulateOptions } from './drivers';
 import { EntityData, EntityMetadata, EntityName, AnyEntity, IPrimaryKey, FilterQuery, Primary, Dictionary } from './typings';
 import { QueryOrderMap } from './enums';
 import { MetadataStorage } from './metadata';
@@ -83,7 +83,9 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
     this.validator.validateParams(where);
     const options = Utils.isObject<FindOptions<T>>(populate) ? populate : { populate, orderBy, limit, offset };
     options.orderBy = options.orderBy || {};
-    const results = await this.driver.find<T>(entityName, where, { ...options, populate: this.preparePopulate(options.populate) }, this.transactionContext);
+    const preparedPopulate = this.preparePopulate(entityName, options.populate);
+    const opts = { ...options, populate: preparedPopulate };
+    const results = await this.driver.find<T>(entityName, where, opts, this.transactionContext);
 
     if (results.length === 0) {
       return [];
@@ -97,7 +99,7 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
     }
 
     const unique = Utils.unique(ret);
-    await this.entityLoader.populate<T>(entityName, unique, options.populate || [], where, options.orderBy, options.refresh);
+    await this.entityLoader.populate<T>(entityName, unique, preparedPopulate, where, options.orderBy, options.refresh);
 
     return unique;
   }
@@ -155,7 +157,10 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
     }
 
     this.validator.validateParams(where);
-    const data = await this.driver.findOne(entityName, where, { ...options, populate: this.preparePopulate(options.populate) }, this.transactionContext);
+    const preparedPopulate = this.preparePopulate(entityName, options.populate);
+    options.populate = preparedPopulate;
+
+    const data = await this.driver.findOne(entityName, where, { ...options, populate: preparedPopulate }, this.transactionContext);
 
     if (!data) {
       return null;
@@ -494,15 +499,17 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
     return ret;
   }
 
-  async populate<T extends AnyEntity<T>, K extends T | T[]>(entities: K, populate: string | string[] | boolean, where: FilterQuery<T> = {}, orderBy: QueryOrderMap = {}, refresh = false, validate = true): Promise<K> {
+  async populate<T extends AnyEntity<T>, K extends T | T[]>(entities: K, populate: string | Populate, where: FilterQuery<T> = {}, orderBy: QueryOrderMap = {}, refresh = false, validate = true): Promise<K> {
     const entitiesArray = Utils.asArray(entities);
 
     if (entitiesArray.length === 0) {
       return entities;
     }
+    populate = typeof populate === 'string' ? Utils.asArray(populate) : populate;
 
     const entityName = entitiesArray[0].constructor.name;
-    await this.entityLoader.populate(entityName, entitiesArray, populate, where, orderBy, refresh, validate);
+    const preparedPopulate = this.preparePopulate(entityName, populate);
+    await this.entityLoader.populate(entityName, entitiesArray, preparedPopulate, where, orderBy, refresh, validate);
 
     return entities;
   }
@@ -579,13 +586,32 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
       await this.lock(entity, options.lockMode, options.lockVersion);
     }
 
-    await this.entityLoader.populate(entityName, [entity], options.populate || [], where, options.orderBy || {}, options.refresh);
+    const preparedPopulate = this.preparePopulate(entityName, options.populate);
+
+    await this.entityLoader.populate(entityName, [entity], preparedPopulate, where, options.orderBy || {}, options.refresh);
 
     return entity;
   }
 
-  private preparePopulate(populate?: string[] | boolean) {
-    return Array.isArray(populate) ? populate : [];
+  private preparePopulate(entityName: string, populate?: Populate): PopulateOptions[] {
+    if (!populate) {
+      return [];
+    }
+
+    if (populate === true) {
+      return [{ field: '*', all: true }];
+    }
+
+    const meta = this.metadata.get(entityName);
+
+    return populate.map(field => {
+      if (Utils.isString(field)) {
+        const strategy = meta.properties[field]?.strategy;
+        return { field, strategy };
+      }
+
+      return field;
+    });
   }
 
 }

packages/core/src/decorators/Property.ts

@@ -1,6 +1,6 @@
 import { MetadataStorage } from '../metadata';
 import { Utils } from '../utils';
-import { Cascade, EntityValidator, ReferenceType } from '../entity';
+import { Cascade, EntityValidator, ReferenceType, LoadStrategy } from '../entity';
 import { EntityName, EntityProperty, AnyEntity, Constructor } from '../typings';
 import { Type } from '../types';
 
@@ -60,4 +60,5 @@ export interface ReferenceOptions<T extends AnyEntity<T>, O extends AnyEntity<O>
   entity?: string | (() => EntityName<T>);
   cascade?: Cascade[];
   eager?: boolean;
+  strategy?: LoadStrategy;
 }

packages/core/src/drivers/DatabaseDriver.ts

@@ -1,4 +1,4 @@
-import { EntityManagerType, FindOneOptions, FindOptions, IDatabaseDriver } from './IDatabaseDriver';
+import { EntityManagerType, FindOneOptions, FindOptions, IDatabaseDriver, PopulateOptions } from './IDatabaseDriver';
 import { EntityData, EntityMetadata, EntityProperty, FilterQuery, AnyEntity, Dictionary, Primary } from '../typings';
 import { MetadataStorage } from '../metadata';
 import { Connection, QueryResult, Transaction } from '../connections';
@@ -51,7 +51,7 @@ export abstract class DatabaseDriver<C extends Connection> implements IDatabaseD
     await this.nativeUpdate<T>(coll.owner.constructor.name, wrap(coll.owner, true).__primaryKey, data, ctx);
   }
 
-  mapResult<T extends AnyEntity<T>>(result: EntityData<T>, meta: EntityMetadata): T | null {
+  mapResult<T extends AnyEntity<T>>(result: EntityData<T>, meta: EntityMetadata, populate: PopulateOptions[] = []): T | null {
     if (!result || !meta) {
       return null;
     }

packages/core/src/drivers/IDatabaseDriver.ts

@@ -4,7 +4,7 @@ import { QueryOrderMap, QueryFlag } from '../enums';
 import { Platform } from '../platforms';
 import { MetadataStorage } from '../metadata';
 import { LockMode } from '../unit-of-work';
-import { Collection } from '../entity';
+import { Collection, LoadStrategy } from '../entity';
 import { EntityManager } from '../index';
 import { DriverException } from '../exceptions';
 
@@ -46,7 +46,7 @@ export interface IDatabaseDriver<C extends Connection = Connection> {
 
   aggregate(entityName: string, pipeline: any[]): Promise<any[]>;
 
-  mapResult<T extends AnyEntity<T>>(result: EntityData<T>, meta: EntityMetadata): T | null;
+  mapResult<T extends AnyEntity<T>>(result: EntityData<T>, meta: EntityMetadata, populate?: PopulateOptions[]): T | null;
 
   /**
    * When driver uses pivot tables for M:N, this method will load identifiers for given collections from them
@@ -75,7 +75,7 @@ export interface IDatabaseDriver<C extends Connection = Connection> {
 }
 
 export interface FindOptions<T> {
-  populate?: string[] | boolean;
+  populate?: Populate;
   orderBy?: QueryOrderMap;
   limit?: number;
   offset?: number;
@@ -88,7 +88,7 @@ export interface FindOptions<T> {
 }
 
 export interface FindOneOptions<T> {
-  populate?: string[] | boolean;
+  populate?: Populate;
   orderBy?: QueryOrderMap;
   groupBy?: string | string[];
   having?: QBFilterQuery<T>;
@@ -99,3 +99,11 @@ export interface FindOneOptions<T> {
   schema?: string;
   flags?: QueryFlag[];
 }
+
+export type Populate = (string | PopulateOptions)[] | boolean;
+
+export type PopulateOptions = {
+  field: string;
+  strategy?: LoadStrategy;
+  all?: boolean;
+};

packages/core/src/entity/EntityLoader.ts

@@ -1,11 +1,12 @@
 import { AnyEntity, EntityProperty, FilterQuery } from '../typings';
 import { EntityManager } from '../index';
-import { ReferenceType } from './enums';
+import { ReferenceType, LoadStrategy } from './enums';
 import { Utils, ValidationError } from '../utils';
 import { Collection } from './Collection';
 import { QueryOrder, QueryOrderMap } from '../enums';
 import { Reference } from './Reference';
 import { wrap } from './wrap';
+import { PopulateOptions } from '../drivers';
 
 export class EntityLoader {
 
@@ -14,26 +15,26 @@ export class EntityLoader {
 
   constructor(private readonly em: EntityManager) { }
 
-  async populate<T extends AnyEntity<T>>(entityName: string, entities: T[], populate: string | string[] | boolean, where: FilterQuery<T> = {}, orderBy: QueryOrderMap = {}, refresh = false, validate = true, lookup = true): Promise<void> {
+  async populate<T extends AnyEntity<T>>(entityName: string, entities: T[], populate: PopulateOptions[] | boolean, where: FilterQuery<T> = {}, orderBy: QueryOrderMap = {}, refresh = false, validate = true, lookup = true): Promise<void> {
     if (entities.length === 0 || populate === false) {
       return;
     }
 
     populate = this.normalizePopulate(entityName, populate, lookup);
 
-    const invalid = populate.find(field => !this.em.canPopulate(entityName, field));
+    const invalid = populate.find(({ field }) => !this.em.canPopulate(entityName, field));
 
     if (validate && invalid) {
-      throw ValidationError.invalidPropertyName(entityName, invalid);
+      throw ValidationError.invalidPropertyName(entityName, invalid.field);
     }
 
-    for (const field of populate) {
-      await this.populateField<T>(entityName, entities, field, where, orderBy, refresh);
+    for (const pop of populate) {
+      await this.populateField<T>(entityName, entities, pop.field, where, orderBy, refresh);
     }
   }
 
-  private normalizePopulate(entityName: string, populate: string | string[] | true, lookup: boolean): string[] {
-    if (populate === true) {
+  private normalizePopulate(entityName: string, populate: PopulateOptions[] | true, lookup: boolean): PopulateOptions[] {
+    if (populate === true || populate.some(p => p.all)) {
       populate = this.lookupAllRelationships(entityName);
     } else {
       populate = Utils.asArray(populate);
@@ -144,7 +145,20 @@ export class EntityLoader {
 
     const filtered = Utils.unique(children);
     const prop = this.metadata.get(entityName).properties[f];
-    await this.populate<T>(prop.type, filtered, [parts.join('.')], where[prop.name], orderBy[prop.name] as QueryOrderMap, refresh, false, false);
+
+    await this.populate<T>(
+      prop.type,
+      filtered,
+      [{
+        field: parts.join('.'),
+        strategy: LoadStrategy.SELECT_IN,
+      }],
+      where[prop.name],
+      orderBy[prop.name] as QueryOrderMap,
+      refresh,
+      false,
+      false
+    );
   }
 
   private async findChildrenFromPivotTable<T extends AnyEntity<T>>(filtered: T[], prop: EntityProperty, field: keyof T, refresh: boolean, where?: FilterQuery<T>, orderBy?: QueryOrderMap): Promise<AnyEntity[]> {
@@ -195,13 +209,13 @@ export class EntityLoader {
     return children.filter(e => !wrap(e[field], true).isInitialized()).map(e => Utils.unwrapReference(e[field]));
   }
 
-  private lookupAllRelationships(entityName: string, prefix = '', visited: string[] = []): string[] {
+  private lookupAllRelationships(entityName: string, prefix = '', visited: string[] = []): PopulateOptions[] {
     if (visited.includes(entityName)) {
       return [];
     }
 
     visited.push(entityName);
-    const ret: string[] = [];
+    const ret: PopulateOptions[] = [];
     const meta = this.metadata.get(entityName);
 
     Object.values(meta.properties)
@@ -213,14 +227,17 @@ export class EntityLoader {
         if (nested.length > 0) {
           ret.push(...nested);
         } else {
-          ret.push(prefixed);
+          ret.push({
+            field: prefixed,
+            strategy: LoadStrategy.SELECT_IN,
+          });
         }
       });
 
     return ret;
   }
 
-  private lookupEagerLoadedRelationships(entityName: string, populate: string[], prefix = '', visited: string[] = []): string[] {
+  private lookupEagerLoadedRelationships(entityName: string, populate: PopulateOptions[], prefix = '', visited: string[] = []): PopulateOptions[] {
     if (visited.includes(entityName)) {
       return [];
     }
@@ -237,7 +254,10 @@ export class EntityLoader {
         if (nested.length > 0) {
           populate.push(...nested);
         } else {
-          populate.push(prefixed);
+          populate.push({
+            field: prefixed,
+            strategy: LoadStrategy.SELECT_IN,
+          });
         }
       });
 

packages/core/src/entity/enums.ts

@@ -13,3 +13,8 @@ export enum Cascade {
   REMOVE = 'remove',
   ALL = 'all',
 }
+
+export enum LoadStrategy {
+  SELECT_IN = 'select-in',
+  JOINED = 'joined'
+}

packages/core/src/metadata/EntitySchema.ts

@@ -3,7 +3,7 @@ import {
   EmbeddedOptions, EnumOptions, IndexOptions, ManyToManyOptions, ManyToOneOptions, OneToManyOptions, OneToOneOptions, PrimaryKeyOptions, PropertyOptions,
   SerializedPrimaryKeyOptions, UniqueOptions,
 } from '../decorators';
-import { BaseEntity, Cascade, Collection, EntityRepository, ReferenceType } from '../entity';
+import { BaseEntity, Cascade, Collection, EntityRepository, ReferenceType, LoadStrategy } from '../entity';
 import { Type } from '../types';
 import { Utils } from '../utils';
 
@@ -113,7 +113,7 @@ export class EntitySchema<T extends AnyEntity<T> = AnyEntity, U extends AnyEntit
   }
 
   addManyToOne<K = object>(name: string & keyof T, type: TypeType, options: ManyToOneOptions<K, T>): void {
-    const prop = { reference: ReferenceType.MANY_TO_ONE, cascade: [Cascade.PERSIST, Cascade.MERGE], ...options } as unknown as EntityProperty<T>;
+    const prop = this.createProperty(ReferenceType.MANY_TO_ONE, options);
     Utils.defaultValue(prop, 'nullable', prop.cascade.includes(Cascade.REMOVE) || prop.cascade.includes(Cascade.ALL));
 
     if (prop.joinColumns && !prop.fieldNames) {
@@ -138,17 +138,17 @@ export class EntitySchema<T extends AnyEntity<T> = AnyEntity, U extends AnyEntit
       Utils.renameKey(options, 'mappedBy', 'inversedBy');
     }
 
-    const prop = { reference: ReferenceType.MANY_TO_MANY, cascade: [Cascade.PERSIST, Cascade.MERGE], ...options } as PropertyOptions<T>;
+    const prop = this.createProperty(ReferenceType.MANY_TO_MANY, options);
     this.addProperty(name, type, prop);
   }
 
   addOneToMany<K = object>(name: string & keyof T, type: TypeType, options: OneToManyOptions<K, T>): void {
-    const prop = { reference: ReferenceType.ONE_TO_MANY, cascade: [Cascade.PERSIST, Cascade.MERGE], ...options } as PropertyOptions<T>;
+    const prop = this.createProperty<T>(ReferenceType.ONE_TO_MANY, options);
     this.addProperty(name, type, prop);
   }
 
   addOneToOne<K = object>(name: string & keyof T, type: TypeType, options: OneToOneOptions<K, T>): void {
-    const prop = { reference: ReferenceType.ONE_TO_ONE, cascade: [Cascade.PERSIST, Cascade.MERGE], ...options } as unknown as EntityProperty<T>;
+    const prop = this.createProperty(ReferenceType.ONE_TO_ONE, options) as EntityProperty;
     Utils.defaultValue(prop, 'nullable', prop.cascade.includes(Cascade.REMOVE) || prop.cascade.includes(Cascade.ALL));
     Utils.defaultValue(prop, 'owner', !!prop.inversedBy || !prop.mappedBy);
     Utils.defaultValue(prop, 'unique', prop.owner);
@@ -305,4 +305,13 @@ export class EntitySchema<T extends AnyEntity<T> = AnyEntity, U extends AnyEntit
     return type;
   }
 
+  private createProperty<T>(reference: ReferenceType, options: PropertyOptions<T> | EntityProperty) {
+    return {
+      reference,
+      cascade: [Cascade.PERSIST, Cascade.MERGE],
+      strategy: LoadStrategy.SELECT_IN,
+      ...options,
+    };
+  }
+
 }

packages/core/src/typings.ts

@@ -1,5 +1,5 @@
 import { QueryOrder } from './enums';
-import { AssignOptions, Cascade, Collection, EntityRepository, EntityValidator, IdentifiedReference, Reference, ReferenceType } from './entity';
+import { AssignOptions, Cascade, Collection, EntityRepository, EntityValidator, IdentifiedReference, Reference, ReferenceType, LoadStrategy } from './entity';
 import { EntityManager } from './EntityManager';
 import { LockMode } from './unit-of-work';
 import { Platform } from './platforms';
@@ -135,6 +135,7 @@ export interface EntityProperty<T extends AnyEntity<T> = any> {
   onUpdate?: (entity: T) => any;
   onDelete?: 'cascade' | 'no action' | 'set null' | 'set default' | string;
   onUpdateIntegrity?: 'cascade' | 'no action' | 'set null' | 'set default' | string;
+  strategy?: LoadStrategy;
   owner: boolean;
   inversedBy: string;
   mappedBy: string;

packages/core/src/unit-of-work/ChangeSetPersister.ts

@@ -75,7 +75,11 @@ export class ChangeSetPersister {
     }
 
     if (meta.versionProperty && [ChangeSetType.CREATE, ChangeSetType.UPDATE].includes(changeSet.type)) {
-      const e = await this.driver.findOne<T>(meta.name, wrap(changeSet.entity, true).__primaryKey, { populate: [meta.versionProperty] }, ctx);
+      const e = await this.driver.findOne<T>(meta.name, wrap(changeSet.entity, true).__primaryKey, {
+        populate: [{
+          field: meta.versionProperty,
+        }],
+      }, ctx);
       (changeSet.entity as T)[meta.versionProperty] = e![meta.versionProperty];
     }
   }