diff --git a/docs/docs/nested-populate.md b/docs/docs/populating-relations.md similarity index 75% rename from docs/docs/nested-populate.md rename to docs/docs/populating-relations.md index cf5681f88eaf..e6c1faa2fe31 100644 --- a/docs/docs/nested-populate.md +++ b/docs/docs/populating-relations.md @@ -1,5 +1,5 @@ --- -title: Nested Populate +title: Populating relations --- `MikroORM` is capable of loading large nested structures while maintaining good performance, querying each database table only once. Imagine you have this nested structure: @@ -10,7 +10,9 @@ title: Nested Populate When you use nested populate while querying all `BookTag`s, this is what happens in the background: ```ts -const tags = await em.findAll(BookTag, { populate: ['books.publisher.tests', 'books.author'] }); +const tags = await em.find(BookTag, {}, { + populate: ['books.publisher.tests', 'books.author'], +}); console.log(tags[0].books[0].publisher.tests[0].name); // prints name of nested test console.log(tags[0].books[0].author.name); // prints name of nested author ``` @@ -21,8 +23,6 @@ console.log(tags[0].books[0].author.name); // prints name of nested author 4. Load all `Test`s associated with previously loaded `Publisher`s 5. Load all `Author`s associated with previously loaded `Book`s -> You can also populate all relationships by passing `populate: ['*']`. - For SQL drivers with pivot tables this means: ```sql @@ -53,6 +53,33 @@ db.getCollection("test").find({"_id":{"$in":[...]}}).toArray(); db.getCollection("author").find({"_id":{"$in":[...]}}).toArray(); ``` +## Populating all relations + +You can also populate all relationships by passing `populate: ['*']`. The result will be also strictly typed (the `Loaded` type respects the star hint). + +```ts +const tags = await em.find(BookTag, {}, { + populate: ['*'], +}); +``` + +> This will always use select-in strategy to deal with possible cycles. + +## Inferring populate hint from filter + +If you want to automatically select all the relations that are part of your filter query, use `populate: ['$infer']`: + +```ts +// this will populate all the books and their authors, all via a single query +const tags = await em.find(BookTag, { + books: { author: { name: '...' } }, +}, { + populate: ['$infer'], +}); +``` + +> This will always use joined strategy as we already have the relations joined because they are in the filter. This feature is not available in MongoDB driver as there is no join support. + ## Filter on populated entities The request to populate can be ambiguous. For example, let's say as a hypothetical that there's a `Book` called `'One'` with tags `'Fiction'` and `'Hard Cover'`. @@ -93,7 +120,7 @@ A value provided on a specific query overrides whatever default is specified glo ## Loading strategies -The way that MikroORM fetches the data in a populate is also configurable. By default MikroORM uses a "where in" strategy which runs one separate query for each level of a populate. If you're using an SQL database you can also ask MikroORM to use a join for all tables involved in the populate and run it as a single query. This is again configurable globally or per query. +The way that MikroORM fetches the data based on populate hint is also configurable. By default, MikroORM uses a "select in" strategy which runs one separate query for each level of a populate. If you're using an SQL database you can also ask MikroORM to use a join for all tables involved in the populate and run it as a single query. This is again configurable globally or per query. For more information see the [Loading Strategies section](./loading-strategies.md). diff --git a/docs/docs/query-builder.md b/docs/docs/query-builder.md index 4aa5458aaa81..17424a0c4e35 100644 --- a/docs/docs/query-builder.md +++ b/docs/docs/query-builder.md @@ -158,7 +158,7 @@ console.log(qb.getQuery()); // order by `e1`.`tags` asc ``` -This is currently available only for filtering (`where`) and sorting (`orderBy`), only the root entity will be selected. To populate its relationships, you can use [`em.populate()`](nested-populate.md). If your populated references are _not_ wrapped (methods like `.unwrap()` are `undefined`, make sure that property was defined with `{ wrappedEntity: true }` as described in [Defining Entities](defining-entities.md). +This is currently available only for filtering (`where`) and sorting (`orderBy`), only the root entity will be selected. To populate its relationships, you can use [`em.populate()`](populating-relations.md). ## Explicit Joining diff --git a/docs/sidebars.js b/docs/sidebars.js index 19482c673c6c..360fe68e212e 100755 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -34,10 +34,11 @@ module.exports = { 'unit-of-work', 'identity-map', 'collections', - 'type-safe-relations', 'query-conditions', - 'repositories', + 'populating-relations', + 'type-safe-relations', 'transactions', + 'repositories', 'inheritance-mapping', 'cascading', 'query-builder', @@ -57,7 +58,6 @@ module.exports = { 'deployment', 'caching', 'logging', - 'nested-populate', 'propagation', 'loading-strategies', 'dataloaders', diff --git a/packages/core/src/EntityManager.ts b/packages/core/src/EntityManager.ts index f1a987e50362..7b69891b3518 100644 --- a/packages/core/src/EntityManager.ts +++ b/packages/core/src/EntityManager.ts @@ -40,31 +40,30 @@ import type { } from './drivers'; import type { AnyEntity, + AnyString, AutoPath, ConnectionType, Dictionary, EntityData, EntityDictionary, EntityDTO, + EntityKey, EntityMetadata, EntityName, FilterDef, FilterQuery, + FromEntityType, GetRepository, IHydrator, + IsSubset, Loaded, MaybePromise, + MergeSelected, ObjectQuery, - Populate, PopulateOptions, Primary, - RequiredEntityData, Ref, - EntityKey, - AnyString, - FromEntityType, - IsSubset, - MergeSelected, + RequiredEntityData, } from './typings'; import { EventType, @@ -72,6 +71,7 @@ import { LoadStrategy, LockMode, PopulateHint, + QueryFlag, ReferenceKind, SCALAR_TYPES, type TransactionOptions, @@ -279,6 +279,7 @@ export class EntityManager { return { where: {} as ObjectQuery, populateWhere: options.populateWhere }; } + /* istanbul ignore next */ if (options.populateWhere === PopulateHint.INFER) { return { where, populateWhere: options.populateWhere }; } @@ -369,6 +370,7 @@ export class EntityManager { return children; }; lookUpChildren(children, meta.className); + /* istanbul ignore next */ (where as Dictionary)[meta.root.discriminatorColumn!] = children.length > 0 ? { $in: [meta.discriminatorValue, ...children.map(c => c.discriminatorValue)] } : meta.discriminatorValue; return where; @@ -1864,7 +1866,7 @@ export class EntityManager { Entity extends object, Hint extends string = never, Fields extends string = never, - >(entityName: string, options: Pick, 'populate' | 'strategy' | 'fields'>): PopulateOptions[] { + >(entityName: string, options: Pick, 'populate' | 'strategy' | 'fields' | 'flags'>): PopulateOptions[] { if (options.populate === false) { return []; } @@ -1908,16 +1910,25 @@ export class EntityManager { if (typeof options.populate !== 'boolean') { options.populate = Utils.asArray(options.populate).map(field => { + /* istanbul ignore next */ if (typeof field === 'boolean') { - return { field: meta.primaryKeys[0], strategy: options.strategy, all: field }; + return [{ field: meta.primaryKeys[0], strategy: options.strategy, all: field }]; + } + + // will be handled in QueryBuilder when processing the where condition via CriteriaNode + if (field === '$infer') { + options.flags ??= []; + options.flags.push(QueryFlag.INFER_POPULATE); + + return []; } if (Utils.isString(field)) { - return { field, strategy: options.strategy }; + return [{ field, strategy: options.strategy }]; } - return field; - }) as any; + return [field]; + }).flat() as any; } const ret: PopulateOptions[] = this.entityLoader.normalizePopulate(entityName, options.populate as true, options.strategy as LoadStrategy); diff --git a/packages/core/src/enums.ts b/packages/core/src/enums.ts index 8443cdbf17b6..7b0933a4f802 100644 --- a/packages/core/src/enums.ts +++ b/packages/core/src/enums.ts @@ -97,6 +97,7 @@ export enum QueryFlag { CONVERT_CUSTOM_TYPES = 'CONVERT_CUSTOM_TYPES', INCLUDE_LAZY_FORMULAS = 'INCLUDE_LAZY_FORMULAS', AUTO_JOIN_ONE_TO_ONE_OWNER = 'AUTO_JOIN_ONE_TO_ONE_OWNER', + INFER_POPULATE = 'INFER_POPULATE', } export const SCALAR_TYPES = ['string', 'number', 'boolean', 'Date', 'Buffer', 'RegExp']; diff --git a/packages/core/src/typings.ts b/packages/core/src/typings.ts index ca734f159679..501229361691 100644 --- a/packages/core/src/typings.ts +++ b/packages/core/src/typings.ts @@ -793,7 +793,7 @@ export type FilterDef = { args?: boolean; }; -export type Populate = readonly AutoPath[] | false; +export type Populate = readonly AutoPath[] | false; export type PopulateOptions = { field: EntityKey; diff --git a/packages/knex/src/AbstractSqlDriver.ts b/packages/knex/src/AbstractSqlDriver.ts index f52285ba9b03..b8acb0da91d5 100644 --- a/packages/knex/src/AbstractSqlDriver.ts +++ b/packages/knex/src/AbstractSqlDriver.ts @@ -94,6 +94,7 @@ export abstract class AbstractSqlDriver[]); const joinedPropsOrderBy = this.buildJoinedPropsOrderBy(entityName, qb, meta, joinedProps); const orderBy = [...Utils.asArray(options.orderBy), ...joinedPropsOrderBy]; + Utils.asArray(options.flags).forEach(flag => qb.setFlag(flag)); if (Utils.isPrimaryKey(where, meta.compositePK)) { where = { [Utils.getPrimaryKeyHash(meta.primaryKeys)]: where } as FilterQuery; @@ -127,7 +128,6 @@ export abstract class AbstractSqlDriver qb.setFlag(flag)); const result = await this.rethrow(qb.execute('all')); if (isCursorPagination && !first && !!last) { diff --git a/packages/knex/src/query/ObjectCriteriaNode.ts b/packages/knex/src/query/ObjectCriteriaNode.ts index b6364af368b0..7d7f9796a351 100644 --- a/packages/knex/src/query/ObjectCriteriaNode.ts +++ b/packages/knex/src/query/ObjectCriteriaNode.ts @@ -2,6 +2,7 @@ import { ALIAS_REPLACEMENT, type Dictionary, type EntityKey, + QueryFlag, raw, RawQueryFragment, ReferenceKind, @@ -183,12 +184,17 @@ export class ObjectCriteriaNode extends CriteriaNode { const operator = Utils.isPlainObject(this.payload) && Object.keys(this.payload).every(k => Utils.isOperator(k, false)); const field = `${alias}.${this.prop!.name}`; + const method = qb.hasFlag(QueryFlag.INFER_POPULATE) ? 'joinAndSelect' : 'join'; + if (this.prop!.kind === ReferenceKind.MANY_TO_MANY && (scalar || operator)) { qb.join(field, nestedAlias, undefined, JoinType.pivotJoin, this.getPath()); } else { const prev = qb._fields?.slice(); - qb.join(field, nestedAlias, undefined, JoinType.leftJoin, this.getPath()); - qb._fields = prev; + qb[method](field, nestedAlias, undefined, JoinType.leftJoin, this.getPath()); + + if (!qb.hasFlag(QueryFlag.INFER_POPULATE)) { + qb._fields = prev; + } } return nestedAlias; diff --git a/packages/knex/src/query/QueryBuilder.ts b/packages/knex/src/query/QueryBuilder.ts index ba36eaf88de5..26a74c30cd7d 100644 --- a/packages/knex/src/query/QueryBuilder.ts +++ b/packages/knex/src/query/QueryBuilder.ts @@ -255,7 +255,6 @@ export class QueryBuilder { this._joins[`${fromAlias}.${prop.name}#${alias}`].subquery = subquery; } - this.addSelect(this.getFieldsForJoinedLoad(prop, alias, fields)); const populate = this._joinedProps.get(fromAlias); const item = { field: prop.name, strategy: LoadStrategy.JOINED, children: [] }; @@ -266,6 +265,7 @@ export class QueryBuilder { } this._joinedProps.set(alias, item); + this.addSelect(this.getFieldsForJoinedLoad(prop, alias, fields)); return this as SelectQueryBuilder; } @@ -288,8 +288,28 @@ export class QueryBuilder { protected getFieldsForJoinedLoad(prop: EntityProperty, alias: string, explicitFields?: string[]): Field[] { const fields: Field[] = []; + const populate: PopulateOptions[] = []; + const joinKey = Object.keys(this._joins).find(join => join.endsWith(`#${alias}`)); + + if (joinKey) { + const path = this._joins[joinKey].path!.split('.').slice(1); + let children = this._populate; + + for (let i = 0; i < path.length; i++) { + const child = children.filter(hint => hint.field === path[i]); + + if (child.length === 0) { + break; + } + + children = child.flatMap(c => c.children) as any; + } + + populate.push(...children); + } + prop.targetMeta!.props - .filter(prop => explicitFields ? explicitFields.includes(prop.name) || prop.primary : this.platform.shouldHaveColumn(prop, this._populate)) + .filter(prop => explicitFields ? explicitFields.includes(prop.name) || prop.primary : this.platform.shouldHaveColumn(prop, populate)) .forEach(prop => fields.push(...this.driver.mapPropToFieldNames(this, prop, alias))); return fields; @@ -440,7 +460,7 @@ export class QueryBuilder { } returning(fields?: Field | Field[]): this { - this._returning = fields != null ? Utils.asArray(fields) : fields; + this._returning = Utils.asArray(fields); return this; } @@ -510,6 +530,10 @@ export class QueryBuilder { return this; } + hasFlag(flag: QueryFlag): boolean { + return this.flags.has(flag); + } + cache(config: boolean | number | [string, number] = true): this { this.ensureNotFinalized(); this._cache = config; diff --git a/packages/knex/src/typings.ts b/packages/knex/src/typings.ts index ddf1a087110c..b89443bbddc2 100644 --- a/packages/knex/src/typings.ts +++ b/packages/knex/src/typings.ts @@ -1,5 +1,15 @@ import type { Knex } from 'knex'; -import type { CheckCallback, Dictionary, EntityProperty, GroupOperator, RawQueryFragment, QBFilterQuery, QueryOrderMap, Type } from '@mikro-orm/core'; +import type { + CheckCallback, + Dictionary, + EntityProperty, + GroupOperator, + RawQueryFragment, + QBFilterQuery, + QueryOrderMap, + Type, + QueryFlag, +} from '@mikro-orm/core'; import type { JoinType, QueryType } from './query/enums'; import type { DatabaseSchema, DatabaseTable } from './schema'; @@ -155,6 +165,9 @@ export interface IQueryBuilder { getAliasForJoinPath(path: string): string | undefined; getNextAlias(entityName?: string): string; clone(reset?: boolean): IQueryBuilder; + setFlag(flag: QueryFlag): this; + unsetFlag(flag: QueryFlag): this; + hasFlag(flag: QueryFlag): boolean; } export interface ICriteriaNode { diff --git a/tests/EntityManager.postgre.test.ts b/tests/EntityManager.postgre.test.ts index b72779ebac8c..d011384e7760 100644 --- a/tests/EntityManager.postgre.test.ts +++ b/tests/EntityManager.postgre.test.ts @@ -1807,6 +1807,88 @@ describe('EntityManagerPostgre', () => { 'where "b0"."author_id" is not null and "a3"."name" = $1'); }); + test('populate: $infer', async () => { + const author = new Author2('Jon Snow', 'snow@wall.st'); + const book1 = new Book2('My Life on The Wall, part 1', author); + book1.perex = ref('asd 1'); + const book2 = new Book2('My Life on The Wall, part 2', author); + book2.perex = ref('asd 2'); + const book3 = new Book2('My Life on The Wall, part 3', author); + book3.perex = ref('asd 3'); + const t1 = Test2.create('t1'); + t1.book = book1; + const t2 = Test2.create('t2'); + t2.book = book2; + const t3 = Test2.create('t3'); + t3.book = book3; + author.books.add(book1, book2, book3); + await orm.em.persistAndFlush([author, t1, t2, t3]); + author.favouriteBook = book3; + await orm.em.flush(); + orm.em.clear(); + + const mock = mockLogger(orm, ['query']); + const res2 = await orm.em.find(Book2, { author: { favouriteBook: { author: { name: 'Jon Snow' } } } }, { populate: ['perex', '$infer'] }); + expect(res2).toHaveLength(3); + expect(mock.mock.calls.length).toBe(1); + expect(wrap(res2[0]).toObject()).toMatchObject({ + title: 'My Life on The Wall, part 1', + perex: 'asd 1', + author: { + id: 1, + name: 'Jon Snow', + email: 'snow@wall.st', + favouriteBook: { + title: 'My Life on The Wall, part 3', + perex: 'asd 3', + author: { + name: 'Jon Snow', + email: 'snow@wall.st', + }, + }, + }, + }); + expect(mock.mock.calls[0][0]).toMatch('select "b0".*, "b0".price * 1.19 as "price_taxed", ' + + '"a1"."id" as "a1__id", "a1"."created_at" as "a1__created_at", "a1"."updated_at" as "a1__updated_at", "a1"."name" as "a1__name", "a1"."email" as "a1__email", "a1"."age" as "a1__age", "a1"."terms_accepted" as "a1__terms_accepted", "a1"."optional" as "a1__optional", "a1"."identities" as "a1__identities", "a1"."born" as "a1__born", "a1"."born_time" as "a1__born_time", "a1"."favourite_book_uuid_pk" as "a1__favourite_book_uuid_pk", "a1"."favourite_author_id" as "a1__favourite_author_id", "a1"."identity" as "a1__identity", ' + + '"b2"."uuid_pk" as "b2__uuid_pk", "b2"."created_at" as "b2__created_at", "b2"."title" as "b2__title", "b2"."price" as "b2__price", "b2".price * 1.19 as "b2__price_taxed", "b2"."double" as "b2__double", "b2"."meta" as "b2__meta", "b2"."author_id" as "b2__author_id", "b2"."publisher_id" as "b2__publisher_id", ' + + '"a3"."id" as "a3__id", "a3"."created_at" as "a3__created_at", "a3"."updated_at" as "a3__updated_at", "a3"."name" as "a3__name", "a3"."email" as "a3__email", "a3"."age" as "a3__age", "a3"."terms_accepted" as "a3__terms_accepted", "a3"."optional" as "a3__optional", "a3"."identities" as "a3__identities", "a3"."born" as "a3__born", "a3"."born_time" as "a3__born_time", "a3"."favourite_book_uuid_pk" as "a3__favourite_book_uuid_pk", "a3"."favourite_author_id" as "a3__favourite_author_id", "a3"."identity" as "a3__identity" ' + + 'from "book2" as "b0" ' + + 'left join "author2" as "a1" on "b0"."author_id" = "a1"."id" ' + + 'left join "book2" as "b2" on "a1"."favourite_book_uuid_pk" = "b2"."uuid_pk" ' + + 'left join "author2" as "a3" on "b2"."author_id" = "a3"."id" ' + + 'where "b0"."author_id" is not null and "a3"."name" = $1'); + + orm.em.clear(); + mock.mock.calls.length = 0; + const res4 = await orm.em.find(Book2, { author: { favouriteBook: { $or: [{ author: { name: 'Jon Snow' } }] } } }, { populate: ['$infer'] }); + expect(res4).toHaveLength(3); + expect(mock.mock.calls.length).toBe(1); + expect(mock.mock.calls[0][0]).toMatch('select "b0"."uuid_pk", "b0"."created_at", "b0"."title", "b0"."price", "b0"."double", "b0"."meta", "b0"."author_id", "b0"."publisher_id", "b0".price * 1.19 as "price_taxed", ' + + '"a1"."id" as "a1__id", "a1"."created_at" as "a1__created_at", "a1"."updated_at" as "a1__updated_at", "a1"."name" as "a1__name", "a1"."email" as "a1__email", "a1"."age" as "a1__age", "a1"."terms_accepted" as "a1__terms_accepted", "a1"."optional" as "a1__optional", "a1"."identities" as "a1__identities", "a1"."born" as "a1__born", "a1"."born_time" as "a1__born_time", "a1"."favourite_book_uuid_pk" as "a1__favourite_book_uuid_pk", "a1"."favourite_author_id" as "a1__favourite_author_id", "a1"."identity" as "a1__identity", ' + + '"b2"."uuid_pk" as "b2__uuid_pk", "b2"."created_at" as "b2__created_at", "b2"."title" as "b2__title", "b2"."price" as "b2__price", "b2".price * 1.19 as "b2__price_taxed", "b2"."double" as "b2__double", "b2"."meta" as "b2__meta", "b2"."author_id" as "b2__author_id", "b2"."publisher_id" as "b2__publisher_id", ' + + '"a3"."id" as "a3__id", "a3"."created_at" as "a3__created_at", "a3"."updated_at" as "a3__updated_at", "a3"."name" as "a3__name", "a3"."email" as "a3__email", "a3"."age" as "a3__age", "a3"."terms_accepted" as "a3__terms_accepted", "a3"."optional" as "a3__optional", "a3"."identities" as "a3__identities", "a3"."born" as "a3__born", "a3"."born_time" as "a3__born_time", "a3"."favourite_book_uuid_pk" as "a3__favourite_book_uuid_pk", "a3"."favourite_author_id" as "a3__favourite_author_id", "a3"."identity" as "a3__identity" ' + + 'from "book2" as "b0" ' + + 'left join "author2" as "a1" on "b0"."author_id" = "a1"."id" ' + + 'left join "book2" as "b2" on "a1"."favourite_book_uuid_pk" = "b2"."uuid_pk" ' + + 'left join "author2" as "a3" on "b2"."author_id" = "a3"."id" ' + + 'where "b0"."author_id" is not null and "a3"."name" = $1'); + expect(wrap(res4[0]).toObject()).toMatchObject({ + title: 'My Life on The Wall, part 1', + author: { + id: 1, + name: 'Jon Snow', + email: 'snow@wall.st', + favouriteBook: { + title: 'My Life on The Wall, part 3', + author: { + name: 'Jon Snow', + email: 'snow@wall.st', + }, + }, + }, + }); + }); + test('datetime is stored in correct timezone', async () => { const author = new Author2('n', 'e'); author.createdAt = new Date('2000-01-01T00:00:00Z'); diff --git a/tests/QueryBuilder.test.ts b/tests/QueryBuilder.test.ts index 387b057e989a..658af7c1e7ad 100644 --- a/tests/QueryBuilder.test.ts +++ b/tests/QueryBuilder.test.ts @@ -260,6 +260,7 @@ describe('QueryBuilder', () => { .populate([{ field: 'asd' }]) .setFlag(QueryFlag.AUTO_JOIN_ONE_TO_ONE_OWNER) .limit(2, 1); + expect(qb.hasFlag(QueryFlag.AUTO_JOIN_ONE_TO_ONE_OWNER)).toBe(true); const sql = 'select `fz`.*, `e1`.`id` as `bar_id` from `foo_baz2` as `fz` left join `foo_bar2` as `e1` on `fz`.`id` = `e1`.`baz_id` limit ? offset ?'; expect(qb.getQuery()).toEqual(sql); expect(qb.getParams()).toEqual([2, 1]);