Skip to content

Commit

Permalink
feat(core): implement partial loading support for joined loading stra…
Browse files Browse the repository at this point in the history 
…tegy

Also changes `em.populate()` signature, adding options parameter, allowing to configure
some additional things like loading strategy.

BREAKING CHANGE:
Signature of `em.populate()` changed, it now uses options parameter.

```diff
-populate<P>(entities: T, populate: P, where?: FilterQuery<T>, orderBy?: QueryOrderMap,
-            refresh?: boolean, validate?: boolean): Promise<Loaded<T, P>>;
+populate<P>(entities: T,populate: P, options?: EntityLoaderOptions<T>): Promise<Loaded<T, P>>;
```

Closes #1707
  • Loading branch information
@B4nan
B4nan committed Apr 25, 2021
1 parent 00b54b4 commit 2bebb5e
Show file tree
Hide file tree
Showing 8 changed files with 103 additions and 45 deletions.
22 changes: 13 additions & 9 deletions packages/core/src/EntityManager.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { inspect } from 'util';

import { Configuration, QueryHelper, TransactionContext, Utils } from './utils';
import { AssignOptions, EntityAssigner, EntityFactory, EntityLoader, EntityRepository, EntityValidator, IdentifiedReference, Reference } from './entity';
import { AssignOptions, EntityAssigner, EntityFactory, EntityLoader, EntityLoaderOptions, EntityRepository, EntityValidator, IdentifiedReference, Reference } from './entity';
import { UnitOfWork } from './unit-of-work';
import { CountOptions, DeleteOptions, EntityManagerType, FindOneOptions, FindOneOrFailOptions, FindOptions, IDatabaseDriver, UpdateOptions } from './drivers';
import { AnyEntity, Dictionary, EntityData, EntityDictionary, EntityDTO, EntityMetadata, EntityName, FilterDef, FilterQuery, GetRepository, Loaded, New, Populate, PopulateMap, PopulateOptions, Primary } from './typings';
Expand Down Expand Up @@ -735,22 +735,22 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
/**
* Loads specified relations in batch. This will execute one query for each relation, that will populate it on all of the specified entities.
*/
async populate<T extends AnyEntity<T>, P extends string | keyof T | Populate<T>>(entities: T, populate: P, where?: FilterQuery<T>, orderBy?: QueryOrderMap, refresh?: boolean, validate?: boolean): Promise<Loaded<T, P>>;
async populate<T extends AnyEntity<T>, P extends string | keyof T | Populate<T>>(entities: T, populate: P, options?: EntityLoaderOptions<T>): Promise<Loaded<T, P>>;

/**
* Loads specified relations in batch. This will execute one query for each relation, that will populate it on all of the specified entities.
*/
async populate<T extends AnyEntity<T>, P extends string | keyof T | Populate<T>>(entities: T[], populate: P, where?: FilterQuery<T>, orderBy?: QueryOrderMap, refresh?: boolean, validate?: boolean): Promise<Loaded<T, P>[]>;
async populate<T extends AnyEntity<T>, P extends string | keyof T | Populate<T>>(entities: T[], populate: P, options?: EntityLoaderOptions<T>): Promise<Loaded<T, P>[]>;

/**
* Loads specified relations in batch. This will execute one query for each relation, that will populate it on all of the specified entities.
*/
async populate<T extends AnyEntity<T>, P extends string | keyof T | Populate<T>>(entities: T | T[], populate: P, where?: FilterQuery<T>, orderBy?: QueryOrderMap, refresh?: boolean, validate?: boolean): Promise<Loaded<T, P> | Loaded<T, P>[]>;
async populate<T extends AnyEntity<T>, P extends string | keyof T | Populate<T>>(entities: T | T[], populate: P, options?: EntityLoaderOptions<T>): Promise<Loaded<T, P> | Loaded<T, P>[]>;

/**
* Loads specified relations in batch. This will execute one query for each relation, that will populate it on all of the specified entities.
*/
async populate<T extends AnyEntity<T>, P extends string | keyof T | Populate<T>>(entities: T | T[], populate: P, where: FilterQuery<T> = {}, orderBy: QueryOrderMap = {}, refresh = false, validate = true): Promise<Loaded<T, P> | Loaded<T, P>[]> {
async populate<T extends AnyEntity<T>, P extends string | keyof T | Populate<T>>(entities: T | T[], populate: P, options: EntityLoaderOptions<T> = {}): Promise<Loaded<T, P> | Loaded<T, P>[]> {
const entitiesArray = Utils.asArray(entities);

if (entitiesArray.length === 0) {
Expand All @@ -761,7 +761,11 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {

const entityName = entitiesArray[0].constructor.name;
const preparedPopulate = this.preparePopulate<T>(entityName, populate as true);
await this.entityLoader.populate(entityName, entitiesArray, preparedPopulate, { where, orderBy, refresh, validate, convertCustomTypes: false });
await this.entityLoader.populate(entityName, entitiesArray, preparedPopulate, {
...options,
/* istanbul ignore next */
convertCustomTypes: options.convertCustomTypes ?? false,
});

return entities as Loaded<T, P>[];
}
Expand Down Expand Up @@ -887,7 +891,7 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {

private preparePopulate<T extends AnyEntity<T>>(entityName: string, populate?: Populate<T>, strategy?: LoadStrategy): PopulateOptions<T>[] {
if (!populate) {
return this.entityLoader.normalizePopulate<T>(entityName, []);
return this.entityLoader.normalizePopulate<T>(entityName, [], strategy);
}

const meta = this.metadata.get(entityName);
Expand All @@ -899,14 +903,14 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
if (Array.isArray(populate)) {
populate = (populate as string[]).map(field => {
if (Utils.isString(field)) {
return { field };
return { field, strategy };
}

return field;
}) as unknown as Populate<T>;
}

const ret: PopulateOptions<T>[] = this.entityLoader.normalizePopulate<T>(entityName, populate as true);
const ret: PopulateOptions<T>[] = this.entityLoader.normalizePopulate<T>(entityName, populate as true, strategy);

return ret.map(field => {
field.strategy = strategy ?? field.strategy ?? this.config.get('loadStrategy');
Expand Down
37 changes: 19 additions & 18 deletions packages/core/src/entity/EntityLoader.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { LoadStrategy, QueryOrder, QueryOrderMap, ReferenceType } from '../enums
import { Reference } from './Reference';
import { FieldsMap, FindOptions } from '../drivers/IDatabaseDriver';

type Options<T extends AnyEntity<T>> = {
export type EntityLoaderOptions<T> = {
where?: FilterQuery<T>;
fields?: (string | FieldsMap)[];
orderBy?: QueryOrderMap;
Expand All @@ -17,6 +17,7 @@ type Options<T extends AnyEntity<T>> = {
lookup?: boolean;
convertCustomTypes?: boolean;
filters?: Dictionary<boolean | Dictionary> | string[] | boolean;
strategy?: LoadStrategy;
};

export class EntityLoader {
Expand All @@ -29,7 +30,7 @@ export class EntityLoader {
/**
* Loads specified relations in batch. This will execute one query for each relation, that will populate it on all of the specified entities.
*/
async populate<T extends AnyEntity<T>>(entityName: string, entities: T[], populate: PopulateOptions<T>[] | boolean, options: Options<T>): Promise<void> {
async populate<T extends AnyEntity<T>>(entityName: string, entities: T[], populate: PopulateOptions<T>[] | boolean, options: EntityLoaderOptions<T>): Promise<void> {
if (entities.length === 0 || populate === false) {
return;
}
Expand All @@ -41,7 +42,7 @@ export class EntityLoader {
options.validate = options.validate ?? true;
options.refresh = options.refresh ?? false;
options.convertCustomTypes = options.convertCustomTypes ?? true;
populate = this.normalizePopulate<T>(entityName, populate, options.lookup);
populate = this.normalizePopulate<T>(entityName, populate, options.strategy, options.lookup);
const invalid = populate.find(({ field }) => !this.em.canPopulate(entityName, field));

if (options.validate && invalid) {
Expand All @@ -51,19 +52,19 @@ export class EntityLoader {
entities.forEach(e => e.__helper!.__serializationContext.populate = e.__helper!.__serializationContext.populate ?? populate as PopulateOptions<T>[]);

for (const pop of populate) {
await this.populateField<T>(entityName, entities, pop, options as Required<Options<T>>);
await this.populateField<T>(entityName, entities, pop, options as Required<EntityLoaderOptions<T>>);
}
}

normalizePopulate<T>(entityName: string, populate: PopulateOptions<T>[] | true, lookup = true): PopulateOptions<T>[] {
normalizePopulate<T>(entityName: string, populate: PopulateOptions<T>[] | true, strategy?: LoadStrategy, lookup = true): PopulateOptions<T>[] {
if (populate === true || populate.some(p => p.all)) {
populate = this.lookupAllRelationships(entityName);
populate = this.lookupAllRelationships(entityName, strategy);
} else {
populate = Utils.asArray(populate);
}

if (lookup) {
populate = this.lookupEagerLoadedRelationships(entityName, populate);
populate = this.lookupEagerLoadedRelationships(entityName, populate, strategy);
}

// convert nested `field` with dot syntax to PopulateOptions with children array
Expand Down Expand Up @@ -130,7 +131,7 @@ export class EntityLoader {
/**
* preload everything in one call (this will update already existing references in IM)
*/
private async populateMany<T extends AnyEntity<T>>(entityName: string, entities: T[], populate: PopulateOptions<T>, options: Required<Options<T>>): Promise<AnyEntity[]> {
private async populateMany<T extends AnyEntity<T>>(entityName: string, entities: T[], populate: PopulateOptions<T>, options: Required<EntityLoaderOptions<T>>): Promise<AnyEntity[]> {
const field = populate.field as keyof T;
const meta = this.metadata.find<T>(entityName)!;
const prop = meta.properties[field as string];
Expand Down Expand Up @@ -208,7 +209,7 @@ export class EntityLoader {
}
}

private async findChildren<T extends AnyEntity<T>>(entities: T[], prop: EntityProperty, populate: PopulateOptions<T>, options: Required<Options<T>>): Promise<AnyEntity[]> {
private async findChildren<T extends AnyEntity<T>>(entities: T[], prop: EntityProperty, populate: PopulateOptions<T>, options: Required<EntityLoaderOptions<T>>): Promise<AnyEntity[]> {
const children = this.getChildReferences<T>(entities, prop, options.refresh);
const meta = this.metadata.find(prop.type)!;
let fk = Utils.getPrimaryKeyHash(meta.primaryKeys);
Expand Down Expand Up @@ -241,7 +242,7 @@ export class EntityLoader {
});
}

private async populateField<T extends AnyEntity<T>>(entityName: string, entities: T[], populate: PopulateOptions<T>, options: Required<Options<T>>): Promise<void> {
private async populateField<T extends AnyEntity<T>>(entityName: string, entities: T[], populate: PopulateOptions<T>, options: Required<EntityLoaderOptions<T>>): Promise<void> {
if (!populate.children) {
return void await this.populateMany<T>(entityName, entities, populate, options);
}
Expand Down Expand Up @@ -273,7 +274,7 @@ export class EntityLoader {
});
}

private async findChildrenFromPivotTable<T extends AnyEntity<T>>(filtered: T[], prop: EntityProperty<T>, options: Required<Options<T>>, orderBy?: QueryOrderMap, populate?: PopulateOptions<T>): Promise<AnyEntity[]> {
private async findChildrenFromPivotTable<T extends AnyEntity<T>>(filtered: T[], prop: EntityProperty<T>, options: Required<EntityLoaderOptions<T>>, orderBy?: QueryOrderMap, populate?: PopulateOptions<T>): Promise<AnyEntity[]> {
const ids = filtered.map((e: AnyEntity<T>) => e.__helper!.__primaryKeys);
const refresh = options.refresh;
/* istanbul ignore next */
Expand Down Expand Up @@ -303,7 +304,7 @@ export class EntityLoader {
return children;
}

private buildFields<T>(prop: EntityProperty<T>, options: Required<Options<T>>) {
private buildFields<T>(prop: EntityProperty<T>, options: Required<EntityLoaderOptions<T>>) {
return (options.fields || []).reduce((ret, f) => {
if (Utils.isPlainObject(f)) {
Object.keys(f)
Expand Down Expand Up @@ -359,7 +360,7 @@ export class EntityLoader {
return children.filter(e => !(e[field] as AnyEntity).__helper!.__initialized).map(e => Reference.unwrapReference(e[field]));
}

private lookupAllRelationships<T>(entityName: string, prefix = '', visited: string[] = []): PopulateOptions<T>[] {
private lookupAllRelationships<T>(entityName: string, strategy?: LoadStrategy, prefix = '', visited: string[] = []): PopulateOptions<T>[] {
if (visited.includes(entityName)) {
return [];
}
Expand All @@ -370,22 +371,22 @@ export class EntityLoader {

meta.relations.forEach(prop => {
const prefixed = prefix ? `${prefix}.${prop.name}` : prop.name;
const nested = this.lookupAllRelationships(prop.type, prefixed, visited);
const nested = this.lookupAllRelationships(prop.type, strategy, prefixed, visited);

if (nested.length > 0) {
ret.push(...nested);
} else {
ret.push({
field: prefixed,
strategy: this.em.config.get('loadStrategy'),
strategy: strategy ?? prop.strategy ?? this.em.config.get('loadStrategy'),
});
}
});

return ret;
}

private lookupEagerLoadedRelationships<T>(entityName: string, populate: PopulateOptions<T>[], prefix = '', visited: string[] = []): PopulateOptions<T>[] {
private lookupEagerLoadedRelationships<T>(entityName: string, populate: PopulateOptions<T>[], strategy?: LoadStrategy, prefix = '', visited: string[] = []): PopulateOptions<T>[] {
if (visited.includes(entityName)) {
return [];
}
Expand All @@ -400,14 +401,14 @@ export class EntityLoader {
const prefixed = prefix ? `${prefix}.${prop.name}` : prop.name;
/* istanbul ignore next */
const nestedPopulate = populate.find(p => p.field === prop.name)?.children ?? [];
const nested = this.lookupEagerLoadedRelationships(prop.type, nestedPopulate, prefixed, visited);
const nested = this.lookupEagerLoadedRelationships(prop.type, nestedPopulate, strategy, prefixed, visited);

if (nested.length > 0) {
ret.push(...nested);
} else {
ret.push({
field: prefixed,
strategy: prop.strategy ?? this.em.config.get('loadStrategy'),
strategy: strategy ?? prop.strategy ?? this.em.config.get('loadStrategy'),
});
}
});
Expand Down
11 changes: 6 additions & 5 deletions packages/core/src/entity/EntityRepository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { EntityData, EntityName, AnyEntity, Primary, Populate, Loaded, New, Filt
import { QueryOrderMap } from '../enums';
import { CountOptions, DeleteOptions, FindOneOptions, FindOneOrFailOptions, FindOptions, UpdateOptions } from '../drivers/IDatabaseDriver';
import { IdentifiedReference, Reference } from './Reference';
import { EntityLoaderOptions } from './EntityLoader';

export class EntityRepository<T extends AnyEntity<T>> {

Expand Down Expand Up @@ -227,23 +228,23 @@ export class EntityRepository<T extends AnyEntity<T>> {
/**
* Loads specified relations in batch. This will execute one query for each relation, that will populate it on all of the specified entities.
*/
async populate<P extends string | keyof T | Populate<T>>(entities: T, populate: P, where?: FilterQuery<T>, orderBy?: QueryOrderMap, refresh?: boolean, validate?: boolean): Promise<Loaded<T, P>>;
async populate<P extends string | keyof T | Populate<T>>(entities: T, populate: P, options?: EntityLoaderOptions<T>): Promise<Loaded<T, P>>;

/**
* Loads specified relations in batch. This will execute one query for each relation, that will populate it on all of the specified entities.
*/
async populate<P extends string | keyof T | Populate<T>>(entities: T[], populate: P, where?: FilterQuery<T>, orderBy?: QueryOrderMap, refresh?: boolean, validate?: boolean): Promise<Loaded<T, P>[]>;
async populate<P extends string | keyof T | Populate<T>>(entities: T[], populate: P, options?: EntityLoaderOptions<T>): Promise<Loaded<T, P>[]>;

/**
* Loads specified relations in batch. This will execute one query for each relation, that will populate it on all of the specified entities.
*/
async populate<P extends string | keyof T | Populate<T>>(entities: T | T[], populate: P, where?: FilterQuery<T>, orderBy?: QueryOrderMap, refresh?: boolean, validate?: boolean): Promise<Loaded<T, P> | Loaded<T, P>[]>;
async populate<P extends string | keyof T | Populate<T>>(entities: T | T[], populate: P, options?: EntityLoaderOptions<T>): Promise<Loaded<T, P> | Loaded<T, P>[]>;

/**
* Loads specified relations in batch. This will execute one query for each relation, that will populate it on all of the specified entities.
*/
async populate<P extends string | keyof T | Populate<T>>(entities: T | T[], populate: P, where: FilterQuery<T> = {}, orderBy: QueryOrderMap = {}, refresh = false, validate = true): Promise<Loaded<T, P> | Loaded<T, P>[]> {
return this.em.populate<T, P>(entities, populate, where, orderBy, refresh, validate);
async populate<P extends string | keyof T | Populate<T>>(entities: T | T[], populate: P, options?: EntityLoaderOptions<T>): Promise<Loaded<T, P> | Loaded<T, P>[]> {
return this.em.populate<T, P>(entities, populate, options);
}

/**
Expand Down
25 changes: 19 additions & 6 deletions packages/knex/src/AbstractSqlDriver.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -501,13 +501,25 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
return res;
}

protected getFieldsForJoinedLoad<T extends AnyEntity<T>>(qb: QueryBuilder<T>, meta: EntityMetadata<T>, populate: PopulateOptions<T>[] = [], parentTableAlias?: string, parentJoinPath?: string): Field<T>[] {
protected getFieldsForJoinedLoad<T extends AnyEntity<T>>(qb: QueryBuilder<T>, meta: EntityMetadata<T>, explicitFields?: Field<T>[], populate: PopulateOptions<T>[] = [], parentTableAlias?: string, parentJoinPath?: string): Field<T>[] {
const fields: Field<T>[] = [];
const joinedProps = this.joinedProps(meta, populate);

const shouldHaveColumn = <U>(prop: EntityProperty<U>, populate: PopulateOptions<U>[], fields?: Field<U>[]) => {
if (!this.shouldHaveColumn(prop, populate)) {
return false;
}

if (!fields || prop.primary) {
return true;
}

return fields.includes(prop.name);
};

// alias all fields in the primary table
meta.props
.filter(prop => this.shouldHaveColumn(prop, populate))
.filter(prop => shouldHaveColumn(prop, populate, explicitFields))
.forEach(prop => fields.push(...this.mapPropToFieldNames(qb, prop, parentTableAlias)));

joinedProps.forEach(relation => {
Expand All @@ -517,7 +529,8 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
const field = parentTableAlias ? `${parentTableAlias}.${prop.name}` : prop.name;
const path = parentJoinPath ? `${parentJoinPath}.${prop.name}` : `${meta.name}.${prop.name}`;
qb.join(field, tableAlias, {}, 'leftJoin', path);
fields.push(...this.getFieldsForJoinedLoad(qb, meta2, relation.children, tableAlias, path));
const childExplicitFields = explicitFields?.filter(f => Utils.isPlainObject(f)).map(o => o[prop.name])[0];
fields.push(...this.getFieldsForJoinedLoad(qb, meta2, childExplicitFields, relation.children, tableAlias, path));
});

return fields;
Expand Down Expand Up @@ -663,7 +676,9 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
const hasExplicitFields = !!fields;
const ret: Field<T>[] = [];

if (fields) {
if (joinedProps.length > 0) {
ret.push(...this.getFieldsForJoinedLoad(qb, meta, fields, populate));
} else if (fields) {
for (const field of [...fields]) {
if (Utils.isPlainObject(field) || field.toString().includes('.')) {
continue;
Expand All @@ -673,8 +688,6 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
}

ret.unshift(...meta.primaryKeys.filter(pk => !fields.includes(pk)));
} else if (joinedProps.length > 0) {
ret.push(...this.getFieldsForJoinedLoad(qb, meta, populate));
} else if (lazyProps.filter(p => !p.formula).length > 0) {
const props = meta.props.filter(prop => this.shouldHaveColumn(prop, populate, false));
ret.push(...Utils.flatten(props.filter(p => !lazyProps.includes(p)).map(p => p.fieldNames)));
Expand Down
2 changes: 1 addition & 1 deletion tests/EntityManager.mysql.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -441,7 +441,7 @@ describe('EntityManagerMySql', () => {
const jon = (await authorRepository.findOne({ name: 'Jon Snow' }))!;
await orm.em.populate(jon, ['books', 'favouriteBook']);
const authors = await authorRepository.findAll();
await orm.em.populate(authors, ['books', 'favouriteBook'], { books: '123' });
await orm.em.populate(authors, ['books', 'favouriteBook'], { where: { books: '123' } });
expect(await authorRepository.findOne({ email: 'not existing' })).toBeNull();
await expect(orm.em.populate([], ['books', 'favouriteBook'])).resolves.toEqual([]);

Expand Down
Loading

0 comments on commit 2bebb5e

Please sign in to comment.