feat(core): allow inferring populate hint from filter via `populate: …

…['$infer']` (#4939)

If you want to automatically select all the relations that are part of
your filter query, use `populate: ['$infer']`:

```ts
// this will populate all the books and their authors, all via a single query
const tags = await em.find(BookTag, {
  books: { author: { name: '...' } },
}, {
  populate: ['$infer'],
});
```

> This will always use joined strategy as we already have the relations
joined because they are in the filter.

Closes #1309

docs/docs/nested-populate.md → docs/docs/populating-relations.md

@@ -1,5 +1,5 @@
 ---
-title: Nested Populate
+title: Populating relations
 ---
 
 `MikroORM` is capable of loading large nested structures while maintaining good performance, querying each database table only once. Imagine you have this nested structure:
@@ -10,7 +10,9 @@ title: Nested Populate
 When you use nested populate while querying all `BookTag`s, this is what happens in the background:
 
 ```ts
-const tags = await em.findAll(BookTag, { populate: ['books.publisher.tests', 'books.author'] });
+const tags = await em.find(BookTag, {}, {
+  populate: ['books.publisher.tests', 'books.author'],
+});
 console.log(tags[0].books[0].publisher.tests[0].name); // prints name of nested test
 console.log(tags[0].books[0].author.name); // prints name of nested author
 ```
@@ -21,8 +23,6 @@ console.log(tags[0].books[0].author.name); // prints name of nested author
 4. Load all `Test`s associated with previously loaded `Publisher`s
 5. Load all `Author`s associated with previously loaded `Book`s
 
-> You can also populate all relationships by passing `populate: ['*']`.
-
 For SQL drivers with pivot tables this means:
 
 ```sql
@@ -53,6 +53,33 @@ db.getCollection("test").find({"_id":{"$in":[...]}}).toArray();
 db.getCollection("author").find({"_id":{"$in":[...]}}).toArray();
 ```
 
+## Populating all relations
+
+You can also populate all relationships by passing `populate: ['*']`. The result will be also strictly typed (the `Loaded` type respects the star hint).
+
+```ts
+const tags = await em.find(BookTag, {}, { 
+  populate: ['*'],
+});
+```
+
+> This will always use select-in strategy to deal with possible cycles.
+
+## Inferring populate hint from filter
+
+If you want to automatically select all the relations that are part of your filter query, use `populate: ['$infer']`:
+
+```ts
+// this will populate all the books and their authors, all via a single query
+const tags = await em.find(BookTag, {
+  books: { author: { name: '...' } },
+}, { 
+  populate: ['$infer'],
+});
+```
+
+> This will always use joined strategy as we already have the relations joined because they are in the filter. This feature is not available in MongoDB driver as there is no join support.
+
 ## Filter on populated entities
 
 The request to populate can be ambiguous. For example, let's say as a hypothetical that there's a `Book` called `'One'` with tags `'Fiction'` and `'Hard Cover'`.
@@ -93,7 +120,7 @@ A value provided on a specific query overrides whatever default is specified glo
 
 ## Loading strategies
 
-The way that MikroORM fetches the data in a populate is also configurable. By default MikroORM uses a "where in" strategy which runs one separate query for each level of a populate. If you're using an SQL database you can also ask MikroORM to use a join for all tables involved in the populate and run it as a single query. This is again configurable globally or per query.
+The way that MikroORM fetches the data based on populate hint is also configurable. By default, MikroORM uses a "select in" strategy which runs one separate query for each level of a populate. If you're using an SQL database you can also ask MikroORM to use a join for all tables involved in the populate and run it as a single query. This is again configurable globally or per query.
 
 For more information see the [Loading Strategies section](./loading-strategies.md).
 

docs/docs/query-builder.md

@@ -158,7 +158,7 @@ console.log(qb.getQuery());
 // order by `e1`.`tags` asc
 ```
 
-This is currently available only for filtering (`where`) and sorting (`orderBy`), only the root entity will be selected. To populate its relationships, you can use [`em.populate()`](nested-populate.md). If your populated references are _not_ wrapped (methods like `.unwrap()` are `undefined`, make sure that property was defined with `{ wrappedEntity: true }` as described in [Defining Entities](defining-entities.md).
+This is currently available only for filtering (`where`) and sorting (`orderBy`), only the root entity will be selected. To populate its relationships, you can use [`em.populate()`](populating-relations.md).
 
 ## Explicit Joining
 

docs/sidebars.js

@@ -34,10 +34,11 @@ module.exports = {
         'unit-of-work',
         'identity-map',
         'collections',
-        'type-safe-relations',
         'query-conditions',
-        'repositories',
+        'populating-relations',
+        'type-safe-relations',
         'transactions',
+        'repositories',
         'inheritance-mapping',
         'cascading',
         'query-builder',
@@ -57,7 +58,6 @@ module.exports = {
         'deployment',
         'caching',
         'logging',
-        'nested-populate',
         'propagation',
         'loading-strategies',
         'dataloaders',

packages/core/src/EntityManager.ts

@@ -40,38 +40,38 @@ import type {
 } from './drivers';
 import type {
   AnyEntity,
+  AnyString,
   AutoPath,
   ConnectionType,
   Dictionary,
   EntityData,
   EntityDictionary,
   EntityDTO,
+  EntityKey,
   EntityMetadata,
   EntityName,
   FilterDef,
   FilterQuery,
+  FromEntityType,
   GetRepository,
   IHydrator,
+  IsSubset,
   Loaded,
   MaybePromise,
+  MergeSelected,
   ObjectQuery,
-  Populate,
   PopulateOptions,
   Primary,
-  RequiredEntityData,
   Ref,
-  EntityKey,
-  AnyString,
-  FromEntityType,
-  IsSubset,
-  MergeSelected,
+  RequiredEntityData,
 } from './typings';
 import {
   EventType,
   FlushMode,
   LoadStrategy,
   LockMode,
   PopulateHint,
+  QueryFlag,
   ReferenceKind,
   SCALAR_TYPES,
   type TransactionOptions,
@@ -279,6 +279,7 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
       return { where: {} as ObjectQuery<Entity>, populateWhere: options.populateWhere };
     }
 
+    /* istanbul ignore next */
     if (options.populateWhere === PopulateHint.INFER) {
       return { where, populateWhere: options.populateWhere };
     }
@@ -369,6 +370,7 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
       return children;
     };
     lookUpChildren(children, meta.className);
+    /* istanbul ignore next */
     (where as Dictionary)[meta.root.discriminatorColumn!] = children.length > 0 ? { $in: [meta.discriminatorValue, ...children.map(c => c.discriminatorValue)] } : meta.discriminatorValue;
 
     return where;
@@ -1864,7 +1866,7 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
     Entity extends object,
     Hint extends string = never,
     Fields extends string = never,
-  >(entityName: string, options: Pick<FindOptions<Entity, Hint, Fields>, 'populate' | 'strategy' | 'fields'>): PopulateOptions<Entity>[] {
+  >(entityName: string, options: Pick<FindOptions<Entity, Hint, Fields>, 'populate' | 'strategy' | 'fields' | 'flags'>): PopulateOptions<Entity>[] {
     if (options.populate === false) {
       return [];
     }
@@ -1908,16 +1910,25 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
 
     if (typeof options.populate !== 'boolean') {
       options.populate = Utils.asArray(options.populate).map(field => {
+        /* istanbul ignore next */
         if (typeof field === 'boolean') {
-          return { field: meta.primaryKeys[0], strategy: options.strategy, all: field };
+          return [{ field: meta.primaryKeys[0], strategy: options.strategy, all: field }];
+        }
+
+        // will be handled in QueryBuilder when processing the where condition via CriteriaNode
+        if (field === '$infer') {
+          options.flags ??= [];
+          options.flags.push(QueryFlag.INFER_POPULATE);
+
+          return [];
         }
 
         if (Utils.isString(field)) {
-          return { field, strategy: options.strategy };
+          return [{ field, strategy: options.strategy }];
         }
 
-        return field;
-      }) as any;
+        return [field];
+      }).flat() as any;
     }
 
     const ret: PopulateOptions<Entity>[] = this.entityLoader.normalizePopulate<Entity>(entityName, options.populate as true, options.strategy as LoadStrategy);

packages/core/src/enums.ts

@@ -97,6 +97,7 @@ export enum QueryFlag {
   CONVERT_CUSTOM_TYPES = 'CONVERT_CUSTOM_TYPES',
   INCLUDE_LAZY_FORMULAS = 'INCLUDE_LAZY_FORMULAS',
   AUTO_JOIN_ONE_TO_ONE_OWNER = 'AUTO_JOIN_ONE_TO_ONE_OWNER',
+  INFER_POPULATE = 'INFER_POPULATE',
 }
 
 export const SCALAR_TYPES = ['string', 'number', 'boolean', 'Date', 'Buffer', 'RegExp'];

packages/core/src/typings.ts

@@ -793,7 +793,7 @@ export type FilterDef = {
   args?: boolean;
 };
 
-export type Populate<T, P extends string = never> = readonly AutoPath<T, P, '*'>[] | false;
+export type Populate<T, P extends string = never> = readonly AutoPath<T, P, '*' | '$infer'>[] | false;
 
 export type PopulateOptions<T> = {
   field: EntityKey<T>;

packages/knex/src/AbstractSqlDriver.ts

@@ -94,6 +94,7 @@ export abstract class AbstractSqlDriver<Connection extends AbstractSqlConnection
     const fields = this.buildFields(meta, populate, joinedProps, qb, qb.alias, options.fields as unknown as Field<T>[]);
     const joinedPropsOrderBy = this.buildJoinedPropsOrderBy(entityName, qb, meta, joinedProps);
     const orderBy = [...Utils.asArray(options.orderBy), ...joinedPropsOrderBy];
+    Utils.asArray(options.flags).forEach(flag => qb.setFlag(flag));
 
     if (Utils.isPrimaryKey(where, meta.compositePK)) {
       where = { [Utils.getPrimaryKeyHash(meta.primaryKeys)]: where } as FilterQuery<T>;
@@ -127,7 +128,6 @@ export abstract class AbstractSqlDriver<Connection extends AbstractSqlConnection
       qb.setLockMode(options.lockMode, options.lockTableAliases);
     }
 
-    Utils.asArray(options.flags).forEach(flag => qb.setFlag(flag));
     const result = await this.rethrow(qb.execute('all'));
 
     if (isCursorPagination && !first && !!last) {

packages/knex/src/query/ObjectCriteriaNode.ts

@@ -2,6 +2,7 @@ import {
   ALIAS_REPLACEMENT,
   type Dictionary,
   type EntityKey,
+  QueryFlag,
   raw,
   RawQueryFragment,
   ReferenceKind,
@@ -183,12 +184,17 @@ export class ObjectCriteriaNode<T extends object> extends CriteriaNode<T> {
     const operator = Utils.isPlainObject(this.payload) && Object.keys(this.payload).every(k => Utils.isOperator(k, false));
     const field = `${alias}.${this.prop!.name}`;
 
+    const method = qb.hasFlag(QueryFlag.INFER_POPULATE) ? 'joinAndSelect' : 'join';
+
     if (this.prop!.kind === ReferenceKind.MANY_TO_MANY && (scalar || operator)) {
       qb.join(field, nestedAlias, undefined, JoinType.pivotJoin, this.getPath());
     } else {
       const prev = qb._fields?.slice();
-      qb.join(field, nestedAlias, undefined, JoinType.leftJoin, this.getPath());
-      qb._fields = prev;
+      qb[method](field, nestedAlias, undefined, JoinType.leftJoin, this.getPath());
+
+      if (!qb.hasFlag(QueryFlag.INFER_POPULATE)) {
+        qb._fields = prev;
+      }
     }
 
     return nestedAlias;

packages/knex/src/query/QueryBuilder.ts

@@ -255,7 +255,6 @@ export class QueryBuilder<T extends object = AnyEntity> {
       this._joins[`${fromAlias}.${prop.name}#${alias}`].subquery = subquery;
     }
 
-    this.addSelect(this.getFieldsForJoinedLoad(prop, alias, fields));
     const populate = this._joinedProps.get(fromAlias);
     const item = { field: prop.name, strategy: LoadStrategy.JOINED, children: [] };
 
@@ -266,6 +265,7 @@ export class QueryBuilder<T extends object = AnyEntity> {
     }
 
     this._joinedProps.set(alias, item);
+    this.addSelect(this.getFieldsForJoinedLoad(prop, alias, fields));
 
     return this as SelectQueryBuilder<T>;
   }
@@ -288,8 +288,28 @@ export class QueryBuilder<T extends object = AnyEntity> {
 
   protected getFieldsForJoinedLoad(prop: EntityProperty<T>, alias: string, explicitFields?: string[]): Field<T>[] {
     const fields: Field<T>[] = [];
+    const populate: PopulateOptions<T>[] = [];
+    const joinKey = Object.keys(this._joins).find(join => join.endsWith(`#${alias}`));
+
+    if (joinKey) {
+      const path = this._joins[joinKey].path!.split('.').slice(1);
+      let children = this._populate;
+
+      for (let i = 0; i < path.length; i++) {
+        const child = children.filter(hint => hint.field === path[i]);
+
+        if (child.length === 0) {
+          break;
+        }
+
+        children = child.flatMap(c => c.children) as any;
+      }
+
+      populate.push(...children);
+    }
+
     prop.targetMeta!.props
-      .filter(prop => explicitFields ? explicitFields.includes(prop.name) || prop.primary : this.platform.shouldHaveColumn(prop, this._populate))
+      .filter(prop => explicitFields ? explicitFields.includes(prop.name) || prop.primary : this.platform.shouldHaveColumn(prop, populate))
       .forEach(prop => fields.push(...this.driver.mapPropToFieldNames<T>(this, prop, alias)));
 
     return fields;
@@ -440,7 +460,7 @@ export class QueryBuilder<T extends object = AnyEntity> {
   }
 
   returning(fields?: Field<T> | Field<T>[]): this {
-    this._returning = fields != null ? Utils.asArray(fields) : fields;
+    this._returning = Utils.asArray(fields);
     return this;
   }
 
@@ -510,6 +530,10 @@ export class QueryBuilder<T extends object = AnyEntity> {
     return this;
   }
 
+  hasFlag(flag: QueryFlag): boolean {
+    return this.flags.has(flag);
+  }
+
   cache(config: boolean | number | [string, number] = true): this {
     this.ensureNotFinalized();
     this._cache = config;

packages/knex/src/typings.ts

@@ -1,5 +1,15 @@
 import type { Knex } from 'knex';
-import type { CheckCallback, Dictionary, EntityProperty, GroupOperator, RawQueryFragment, QBFilterQuery, QueryOrderMap, Type } from '@mikro-orm/core';
+import type {
+  CheckCallback,
+  Dictionary,
+  EntityProperty,
+  GroupOperator,
+  RawQueryFragment,
+  QBFilterQuery,
+  QueryOrderMap,
+  Type,
+  QueryFlag,
+} from '@mikro-orm/core';
 import type { JoinType, QueryType } from './query/enums';
 import type { DatabaseSchema, DatabaseTable } from './schema';
 
@@ -155,6 +165,9 @@ export interface IQueryBuilder<T> {
   getAliasForJoinPath(path: string): string | undefined;
   getNextAlias(entityName?: string): string;
   clone(reset?: boolean): IQueryBuilder<T>;
+  setFlag(flag: QueryFlag): this;
+  unsetFlag(flag: QueryFlag): this;
+  hasFlag(flag: QueryFlag): boolean;
 }
 
 export interface ICriteriaNode<T extends object> {