Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Relation id and afterAll hook performance fixes #8169

Merged
merged 5 commits into from
Feb 16, 2022
Merged

Relation id and afterAll hook performance fixes #8169

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c3ae66e
perf: Increase performance when using entities with relation ids
Lennard-Dietz Sep 9, 2021
75e09fd
perf: Increase performance of afterAll hook
Lennard-Dietz Sep 9, 2021
08d4c0b
perf: Increase performance of entities with relation id
Lennard-Dietz Sep 9, 2021
0764634
Merge branch 'master' of https://github.com/Lennard-Dietz/typeorm int…
Lennard-Dietz Nov 24, 2021
fc0bc3d
style: fix indentation and some array operations
Lennard-Dietz Nov 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions src/query-builder/relation-id/RelationIdLoader.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ export class RelationIdLoader {
if (relationIdAttr.queryBuilderFactory)
throw new TypeORMError("Additional condition can not be used with ManyToOne or OneToOne owner relations.");

const duplicates: Array<string> = [];
const duplicates: {[duplicateKey: string]: boolean} = {};
pleerock marked this conversation as resolved.
Show resolved Hide resolved
const results = rawEntities.map(rawEntity => {
const result: ObjectLiteral = {};
const duplicateParts: Array<string> = [];
Expand All @@ -55,10 +55,10 @@ export class RelationIdLoader {

duplicateParts.sort();
const duplicate = duplicateParts.join("::");
if (duplicates.indexOf(duplicate) !== -1) {
if (duplicates[duplicate]) {
return null;
}
duplicates.push(duplicate);
duplicates[duplicate] = true;
return result;
}).filter(v => v);

Expand All @@ -78,7 +78,7 @@ export class RelationIdLoader {
const tableName = relation.inverseEntityMetadata.tableName; // category
const tableAlias = relationIdAttr.alias || tableName; // if condition (custom query builder factory) is set then relationIdAttr.alias defined

const duplicates: Array<string> = [];
const duplicates: {[duplicateKey: string]: boolean} = {};
pleerock marked this conversation as resolved.
Show resolved Hide resolved
const parameters: ObjectLiteral = {};
const condition = rawEntities.map((rawEntity, index) => {
const duplicateParts: Array<string> = [];
Expand All @@ -96,10 +96,10 @@ export class RelationIdLoader {
}).filter(v => v).join(" AND ");
duplicateParts.sort();
const duplicate = duplicateParts.join("::");
if (duplicates.indexOf(duplicate) !== -1) {
if (duplicates[duplicate]) {
return "";
}
duplicates.push(duplicate);
duplicates[duplicate] = true;
Object.assign(parameters, parameterParts);
return queryPart;
}).filter(v => v).map(condition => "(" + condition + ")")
Expand Down Expand Up @@ -174,7 +174,7 @@ export class RelationIdLoader {
return { relationIdAttribute: relationIdAttr, results: [] };

const parameters: ObjectLiteral = {};
const duplicates: Array<string> = [];
const duplicates: {[duplicateKey: string]: boolean} = {};
pleerock marked this conversation as resolved.
Show resolved Hide resolved
const joinColumnConditions = mappedColumns.map((mappedColumn, index) => {
const duplicateParts: Array<string> = [];
const parameterParts: ObjectLiteral = {};
Expand All @@ -191,10 +191,10 @@ export class RelationIdLoader {
}).filter(s => s).join(" AND ");
duplicateParts.sort();
const duplicate = duplicateParts.join("::");
if (duplicates.indexOf(duplicate) !== -1) {
if (duplicates[duplicate]) {
return "";
}
duplicates.push(duplicate);
duplicates[duplicate] = true;
Object.assign(parameters, parameterParts);
return queryPart;
}).filter(s => s);
Expand Down
206 changes: 129 additions & 77 deletions src/query-builder/transformer/RawSqlResultsToEntityTransformer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ import {DriverUtils} from "../../driver/DriverUtils";
*/
export class RawSqlResultsToEntityTransformer {

/**
* Contains a hashmap for every rawRelationIdResults given.
* In the hashmap you will find the idMaps of a result under the hash of this.hashEntityIds for the result.
*/
private relationIdMaps: Array<{ [idHash: string]: any[] }>;

// -------------------------------------------------------------------------
// Constructor
// -------------------------------------------------------------------------
Expand Down Expand Up @@ -131,7 +137,7 @@ export class RawSqlResultsToEntityTransformer {
metadata.columns.forEach(column => {

// if table inheritance is used make sure this column is not child's column
if (metadata.childEntityMetadatas.length > 0 && metadata.childEntityMetadatas.map(metadata => metadata.target).indexOf(column.target) !== -1)
if (metadata.childEntityMetadatas.length > 0 && metadata.childEntityMetadatas.findIndex(childMetadata => childMetadata.target === column.target) !== -1)
return;

const value = rawResults[0][DriverUtils.buildAlias(this.driver, alias.name, column.databaseName)];
Expand Down Expand Up @@ -206,85 +212,48 @@ export class RawSqlResultsToEntityTransformer {

protected transformRelationIds(rawSqlResults: any[], alias: Alias, entity: ObjectLiteral, metadata: EntityMetadata): boolean {
let hasData = false;
this.rawRelationIdResults.forEach(rawRelationIdResult => {
if (rawRelationIdResult.relationIdAttribute.parentAlias !== alias.name)
return;

const relation = rawRelationIdResult.relationIdAttribute.relation;
const valueMap = this.createValueMapFromJoinColumns(relation, rawRelationIdResult.relationIdAttribute.parentAlias, rawSqlResults);
if (valueMap === undefined || valueMap === null)
return;

const idMaps = rawRelationIdResult.results.map(result => {
const entityPrimaryIds = this.extractEntityPrimaryIds(relation, result);
if (OrmUtils.compareIds(entityPrimaryIds, valueMap) === false)
return;

let columns: ColumnMetadata[];
if (relation.isManyToOne || relation.isOneToOneOwner) {
columns = relation.joinColumns.map(joinColumn => joinColumn);
} else if (relation.isOneToMany || relation.isOneToOneNotOwner) {
columns = relation.inverseEntityMetadata.primaryColumns.map(joinColumn => joinColumn);
// columns = relation.inverseRelation!.joinColumns.map(joinColumn => joinColumn.referencedColumn!); //.inverseEntityMetadata.primaryColumns.map(joinColumn => joinColumn);
} else { // ManyToMany
if (relation.isOwning) {
columns = relation.inverseJoinColumns.map(joinColumn => joinColumn);
} else {
columns = relation.inverseRelation!.joinColumns.map(joinColumn => joinColumn);
}
}

const idMap = columns.reduce((idMap, column) => {
let value = result[column.databaseName];
if (relation.isOneToMany || relation.isOneToOneNotOwner) {
if (column.isVirtual && column.referencedColumn && column.referencedColumn.propertyName !== column.propertyName) // if column is a relation
value = column.referencedColumn.createValueMap(value);

return OrmUtils.mergeDeep(idMap, column.createValueMap(value));
} else {
if (column.referencedColumn!.referencedColumn) // if column is a relation
value = column.referencedColumn!.referencedColumn!.createValueMap(value);

return OrmUtils.mergeDeep(idMap, column.referencedColumn!.createValueMap(value));
}
}, {} as ObjectLiteral);

if (columns.length === 1 && rawRelationIdResult.relationIdAttribute.disableMixedMap === false) {
if (relation.isOneToMany || relation.isOneToOneNotOwner) {
return columns[0].getEntityValue(idMap);
} else {
return columns[0].referencedColumn!.getEntityValue(idMap);
}
}
return idMap;
}).filter(result => result !== undefined);

const properties = rawRelationIdResult.relationIdAttribute.mapToPropertyPropertyPath.split(".");
const mapToProperty = (properties: string[], map: ObjectLiteral, value: any): any => {

const property = properties.shift();
if (property && properties.length === 0) {
map[property] = value;
return map;
} else if (property && properties.length > 0) {
mapToProperty(properties, map[property], value);
} else {
return map;
}
};
if (relation.isOneToOne || relation.isManyToOne) {
if (idMaps[0] !== undefined) {
mapToProperty(properties, entity, idMaps[0]);
hasData = true;
}
this.rawRelationIdResults.forEach((rawRelationIdResult, index) => {
if (rawRelationIdResult.relationIdAttribute.parentAlias !== alias.name)
return;

const relation = rawRelationIdResult.relationIdAttribute.relation;
const valueMap = this.createValueMapFromJoinColumns(relation, rawRelationIdResult.relationIdAttribute.parentAlias, rawSqlResults);
if (valueMap === undefined || valueMap === null) {
return;
}

// prepare common data for this call
this.prepareDataForTransformRelationIds();

// Extract idMaps from prepared data by hash
const hash = this.hashEntityIds(relation, valueMap);
const idMaps = this.relationIdMaps[index][hash] || [];

// Map data to properties
const properties = rawRelationIdResult.relationIdAttribute.mapToPropertyPropertyPath.split(".");
const mapToProperty = (properties: string[], map: ObjectLiteral, value: any): any => {
const property = properties.shift();
if (property && properties.length === 0) {
map[property] = value;
return map;
}
if (property && properties.length > 0) {
mapToProperty(properties, map[property], value);
} else {
mapToProperty(properties, entity, idMaps);
if (idMaps.length > 0) {
hasData = true;
}
return map;
}
};
if (relation.isOneToOne || relation.isManyToOne) {
if (idMaps[0] !== undefined) {
mapToProperty(properties, entity, idMaps[0]);
hasData = true;
}
} else {
mapToProperty(properties, entity, idMaps);
hasData = hasData || idMaps.length > 0;
}
});

return hasData;
}

Expand Down Expand Up @@ -371,4 +340,87 @@ export class RawSqlResultsToEntityTransformer {
virtualColumns.forEach(virtualColumn => delete entity[virtualColumn]);
}*/



/** Prepare data to run #transformRelationIds, as a lot of result independent data is needed in every call */
private prepareDataForTransformRelationIds() {

// Return early if the relationIdMaps were already calculated
if(this.relationIdMaps) {
return;
}

// Ensure this prepare function is only called once
this.relationIdMaps = this.rawRelationIdResults.map(rawRelationIdResult => {
const relation = rawRelationIdResult.relationIdAttribute.relation;

// Calculate column metadata
let columns: ColumnMetadata[];
if (relation.isManyToOne || relation.isOneToOneOwner) {
columns = relation.joinColumns;
} else if (relation.isOneToMany || relation.isOneToOneNotOwner) {
columns = relation.inverseEntityMetadata.primaryColumns;
} else {
// ManyToMany
if (relation.isOwning) {
columns = relation.inverseJoinColumns;
} else {
columns = relation.inverseRelation!.joinColumns;
}
}

// Calculate the idMaps for the rawRelationIdResult
return rawRelationIdResult.results.reduce((agg, result) => {
let idMap = columns.reduce((idMap, column) => {
let value = result[column.databaseName];
if (relation.isOneToMany || relation.isOneToOneNotOwner) {
if (column.isVirtual && column.referencedColumn && column.referencedColumn.propertyName !== column.propertyName) {
// if column is a relation
value = column.referencedColumn.createValueMap(value);
}

return OrmUtils.mergeDeep(idMap, column.createValueMap(value));
}
if (column.referencedColumn!.referencedColumn) {
// if column is a relation
value = column.referencedColumn!.referencedColumn!.createValueMap(value);
}

return OrmUtils.mergeDeep(idMap, column.referencedColumn!.createValueMap(value));
}, {} as ObjectLiteral);

if (columns.length === 1 && !rawRelationIdResult.relationIdAttribute.disableMixedMap) {
if (relation.isOneToMany || relation.isOneToOneNotOwner) {
idMap = columns[0].getEntityValue(idMap);
} else {
idMap = columns[0].referencedColumn!.getEntityValue(idMap);
}
pleerock marked this conversation as resolved.
Show resolved Hide resolved
}

// If an idMap is found, set it in the aggregator under the correct hash
if (idMap !== undefined) {
const hash = this.hashEntityIds(relation, result);

if (agg[hash]) {
agg[hash].push(idMap);
} else {
agg[hash] = [idMap];
}
}

return agg;
}, {});
});

}

/**
* Use a simple JSON.stringify to create a simple hash of the primary ids of an entity.
* As this.extractEntityPrimaryIds always creates the primary id object in the same order, if the same relation is
* given, a simple JSON.stringify should be enough to get a unique hash per entity!
*/
private hashEntityIds(relation: RelationMetadata, data: ObjectLiteral) {
const entityPrimaryIds = this.extractEntityPrimaryIds(relation, data);
return JSON.stringify(entityPrimaryIds);
}
}
64 changes: 32 additions & 32 deletions src/subscriber/Broadcaster.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -437,52 +437,52 @@ export class Broadcaster {
* Note: this method has a performance-optimized code organization, do not change code structure.
*/
broadcastLoadEvent(result: BroadcasterResult, metadata: EntityMetadata, entities: ObjectLiteral[]): void {
entities.forEach(entity => {
if (entity instanceof Promise) // todo: check why need this?
return;
// Calculate which subscribers are fitting for the given entity type
const fittingSubscribers = this.queryRunner.connection.subscribers.filter(subscriber => this.isAllowedSubscriber(subscriber, metadata.target) && subscriber.afterLoad);

if (metadata.relations.length || metadata.afterLoadListeners.length || fittingSubscribers.length) {
// todo: check why need this?
const nonPromiseEntities = entities.filter(entity => !(entity instanceof Promise));

// collect load events for all children entities that were loaded with the main entity
if (metadata.relations.length) {
metadata.relations.forEach(relation => {
nonPromiseEntities.forEach(entity => {
// in lazy relations we cannot simply access to entity property because it will cause a getter and a database query
if (relation.isLazy && !entity.hasOwnProperty(relation.propertyName)) return;

// in lazy relations we cannot simply access to entity property because it will cause a getter and a database query
if (relation.isLazy && !entity.hasOwnProperty(relation.propertyName))
return;

const value = relation.getEntityValue(entity);
if (value instanceof Object)
this.broadcastLoadEvent(result, relation.inverseEntityMetadata, Array.isArray(value) ? value : [value]);
const value = relation.getEntityValue(entity);
if (value instanceof Object) this.broadcastLoadEvent(result, relation.inverseEntityMetadata, Array.isArray(value) ? value : [value]);
});
});
}

if (metadata.afterLoadListeners.length) {
metadata.afterLoadListeners.forEach(listener => {
if (listener.isAllowed(entity)) {
const executionResult = listener.execute(entity);
if (executionResult instanceof Promise)
result.promises.push(executionResult);
result.count++;
}
nonPromiseEntities.forEach(entity => {
if (listener.isAllowed(entity)) {
const executionResult = listener.execute(entity);
if (executionResult instanceof Promise) result.promises.push(executionResult);
result.count++;
}
});
});
}

if (this.queryRunner.connection.subscribers.length) {
this.queryRunner.connection.subscribers.forEach(subscriber => {
if (this.isAllowedSubscriber(subscriber, metadata.target) && subscriber.afterLoad) {
const executionResult = subscriber.afterLoad!(entity, {
connection: this.queryRunner.connection,
queryRunner: this.queryRunner,
manager: this.queryRunner.manager,
entity: entity,
metadata: metadata
});
if (executionResult instanceof Promise)
result.promises.push(executionResult);
result.count++;
}
fittingSubscribers.forEach(subscriber => {
nonPromiseEntities.forEach(entity => {
const executionResult = subscriber.afterLoad!(entity, {
entity,
metadata,
connection: this.queryRunner.connection,
queryRunner: this.queryRunner,
manager: this.queryRunner.manager,
});
if (executionResult instanceof Promise) result.promises.push(executionResult);
result.count++;
});
}
});
});
}
}

// -------------------------------------------------------------------------
Expand Down