feat(core): add support for nested partial loading

Closes #221

docs/docs/entity-manager.md

@@ -186,15 +186,48 @@ console.log(author.books[0].tags[0].isInitialized()); // true, because it was po
 
 ### Fetching Partial Entities
 
+> This feature is supported only for `SELECT_IN` loading strategy.
+
 When fetching single entity, you can choose to select only parts of an entity via `options.fields`:
 
-```typescript
+```ts
 const author = await orm.em.findOne(Author, '...', { fields: ['name', 'born'] });
 console.log(author.id); // PK is always selected
 console.log(author.name); // Jon Snow
 console.log(author.email); // undefined
 ```
 
+From v4.4 it is also possible to specify fields for nested relations:
+
+```ts
+const author = await orm.em.findOne(Author, '...', { fields: ['name', 'books.title', 'books.author', 'books.price'] });
+```
+
+Or with an alternative object syntax:
+
+```ts
+const author = await orm.em.findOne(Author, '...', { fields: ['name', { books: ['title', 'author', 'price'] }] });
+```
+
+It is also possible to use multiple levels:
+
+```ts
+const author = await orm.em.findOne(Author, '...', { fields: ['name', { books: ['title', 'price', 'author', { author: ['email'] }] }] });
+```
+
+Primary keys are always selected even if you omit them. On the other hand, you are responsible 
+for selecting the FKs - if you omit such property, the relation might not be loaded properly.
+In the following example the books would not be linked the author, because we did not specify 
+the `books.author` field to be loaded.
+
+```ts
+// this will load both author and book entities, but they won't be connected due to the missing FK in select
+const author = await orm.em.findOne(Author, '...', { fields: ['name', { books: ['title', 'price'] });
+```
+
+> Same problem can occur in mongo with M:N collections - those are stored as array property 
+> on the owning entity, so you need to make sure to mark such properties too.
+
 ### Fetching Paginated Results
 
 If you are going to paginate your results, you can use `em.findAndCount()` that will return

docs/versioned_docs/version-4.3/entity-manager.md

@@ -186,6 +186,8 @@ console.log(author.books[0].tags[0].isInitialized()); // true, because it was po
 
 ### Fetching Partial Entities
 
+> This feature is supported only for `SELECT_IN` loading strategy.
+
 When fetching single entity, you can choose to select only parts of an entity via `options.fields`:
 
 ```typescript

packages/core/src/drivers/DatabaseDriver.ts

@@ -50,7 +50,7 @@ export abstract class DatabaseDriver<C extends Connection> implements IDatabaseD
     throw new Error(`Aggregations are not supported by ${this.constructor.name} driver`);
   }
 
-  async loadFromPivotTable<T extends AnyEntity<T>, O extends AnyEntity<O>>(prop: EntityProperty, owners: Primary<O>[][], where?: FilterQuery<T>, orderBy?: QueryOrderMap, ctx?: Transaction): Promise<Dictionary<T[]>> {
+  async loadFromPivotTable<T extends AnyEntity<T>, O extends AnyEntity<O>>(prop: EntityProperty, owners: Primary<O>[][], where?: FilterQuery<T>, orderBy?: QueryOrderMap, ctx?: Transaction, options?: FindOptions<T>): Promise<Dictionary<T[]>> {
     throw new Error(`${this.constructor.name} does not use pivot tables`);
   }
 

packages/core/src/drivers/IDatabaseDriver.ts

@@ -54,7 +54,7 @@ export interface IDatabaseDriver<C extends Connection = Connection> {
   /**
    * When driver uses pivot tables for M:N, this method will load identifiers for given collections from them
    */
-  loadFromPivotTable<T extends AnyEntity<T>, O extends AnyEntity<O>>(prop: EntityProperty, owners: Primary<O>[][], where?: FilterQuery<T>, orderBy?: QueryOrderMap, ctx?: Transaction): Promise<Dictionary<T[]>>;
+  loadFromPivotTable<T extends AnyEntity<T>, O extends AnyEntity<O>>(prop: EntityProperty, owners: Primary<O>[][], where?: FilterQuery<T>, orderBy?: QueryOrderMap, ctx?: Transaction, options?: FindOptions<T>): Promise<Dictionary<T[]>>;
 
   getPlatform(): Platform;
 
@@ -77,6 +77,8 @@ export interface IDatabaseDriver<C extends Connection = Connection> {
 
 }
 
+export type FieldsMap = { [K: string]: (string | FieldsMap)[] };
+
 export interface FindOptions<T, P extends Populate<T> = Populate<T>> {
   populate?: P;
   orderBy?: QueryOrderMap;
@@ -85,7 +87,7 @@ export interface FindOptions<T, P extends Populate<T> = Populate<T>> {
   offset?: number;
   refresh?: boolean;
   convertCustomTypes?: boolean;
-  fields?: string[];
+  fields?: (string | FieldsMap)[];
   schema?: string;
   flags?: QueryFlag[];
   groupBy?: string | string[];

packages/core/src/entity/EntityLoader.ts

@@ -1,14 +1,16 @@
-import { AnyEntity, Dictionary, EntityProperty, FilterQuery, PopulateOptions } from '../typings';
+import { AnyEntity, Dictionary, EntityProperty, FilterQuery, PopulateOptions, Primary } from '../typings';
 import { EntityManager } from '../EntityManager';
 import { QueryHelper } from '../utils/QueryHelper';
 import { Utils } from '../utils/Utils';
 import { ValidationError } from '../errors';
 import { Collection } from './Collection';
-import { LoadStrategy, ReferenceType, QueryOrder, QueryOrderMap } from '../enums';
+import { LoadStrategy, QueryOrder, QueryOrderMap, ReferenceType } from '../enums';
 import { Reference } from './Reference';
+import { FieldsMap, FindOptions } from '../drivers/IDatabaseDriver';
 
 type Options<T extends AnyEntity<T>> = {
   where?: FilterQuery<T>;
+  fields?: (string | FieldsMap)[];
   orderBy?: QueryOrderMap;
   refresh?: boolean;
   validate?: boolean;
@@ -152,7 +154,7 @@ export class EntityLoader {
     const innerOrderBy = Utils.isObject(options.orderBy[prop.name]) ? options.orderBy[prop.name] as QueryOrderMap : undefined;
 
     if (prop.reference === ReferenceType.MANY_TO_MANY && this.driver.getPlatform().usesPivotTable()) {
-      return this.findChildrenFromPivotTable<T>(filtered, prop, field, options.refresh, options.where[prop.name], innerOrderBy as QueryOrderMap);
+      return this.findChildrenFromPivotTable<T>(filtered, prop, options, innerOrderBy, populate);
     }
 
     const subCond = Utils.isPlainObject(options.where[prop.name]) ? options.where[prop.name] : {};
@@ -227,13 +229,15 @@ export class EntityLoader {
 
     const ids = Utils.unique(children.map(e => Utils.getPrimaryKeyValues(e, e.__meta!.primaryKeys, true)));
     const where = { ...QueryHelper.processWhere({ [fk]: { $in: ids } }, meta.name!, this.metadata, this.driver.getPlatform(), !options.convertCustomTypes), ...(options.where as Dictionary) } as FilterQuery<T>;
+    const fields = this.buildFields(prop, options);
 
     return this.em.find<T>(prop.type, where, {
       orderBy: options.orderBy || prop.orderBy || { [fk]: QueryOrder.ASC },
       refresh: options.refresh,
       filters: options.filters,
       convertCustomTypes: options.convertCustomTypes,
       populate: populate.children,
+      fields: fields.length > 0 ? fields : undefined,
     });
   }
 
@@ -257,38 +261,68 @@ export class EntityLoader {
 
     const filtered = Utils.unique(children);
     const prop = this.metadata.find(entityName)!.properties[populate.field];
+    const fields = this.buildFields(prop, options);
     await this.populate<T>(prop.type, filtered, populate.children, {
       where: options.where[prop.name],
       orderBy: options.orderBy[prop.name] as QueryOrderMap,
       refresh: options.refresh,
+      fields: fields.length > 0 ? fields : undefined,
       filters: options.filters,
       validate: false,
       lookup: false,
     });
   }
 
-  private async findChildrenFromPivotTable<T extends AnyEntity<T>>(filtered: T[], prop: EntityProperty, field: keyof T, refresh: boolean, where?: FilterQuery<T>, orderBy?: QueryOrderMap): Promise<AnyEntity[]> {
+  private async findChildrenFromPivotTable<T extends AnyEntity<T>>(filtered: T[], prop: EntityProperty<T>, options: Required<Options<T>>, orderBy?: QueryOrderMap, populate?: PopulateOptions<T>): Promise<AnyEntity[]> {
     const ids = filtered.map((e: AnyEntity<T>) => e.__helper!.__primaryKeys);
+    const refresh = options.refresh;
+    const where = options.where[prop.name as string];
+    const fields = this.buildFields(prop, options);
+    const options2 = { ...options } as FindOptions<T>;
+    options2.fields = (fields.length > 0 ? fields : undefined) as string[];
+    /* istanbul ignore next */
+    options2.populate = (populate?.children ?? []) as unknown as string[];
 
     if (prop.customType) {
-      ids.forEach((id, idx) => ids[idx] = QueryHelper.processCustomType(prop, id, this.driver.getPlatform()));
+      ids.forEach((id, idx) => ids[idx] = QueryHelper.processCustomType(prop, id as FilterQuery<T>, this.driver.getPlatform()) as Primary<T>[]);
     }
 
-    const map = await this.driver.loadFromPivotTable(prop, ids, where, orderBy, this.em.getTransactionContext());
+    const map = await this.driver.loadFromPivotTable(prop, ids, where, orderBy, this.em.getTransactionContext(), options2);
     const children: AnyEntity[] = [];
 
     for (const entity of filtered) {
       const items = map[entity.__helper!.getSerializedPrimaryKey()].map(item => {
         const entity = this.em.getEntityFactory().create<T>(prop.type, item, { refresh, merge: true, convertCustomTypes: true });
         return this.em.getUnitOfWork().registerManaged(entity, item, refresh);
       });
-      (entity[field] as unknown as Collection<AnyEntity>).hydrate(items);
+      (entity[prop.name] as unknown as Collection<AnyEntity>).hydrate(items);
       children.push(...items);
     }
 
     return children;
   }
 
+  private buildFields<T>(prop: EntityProperty<T>, options: Required<Options<T>>) {
+    return (options.fields || []).reduce((ret, f) => {
+      if (Utils.isPlainObject(f)) {
+        Object.keys(f)
+          .filter(ff => ff === prop.name)
+          .forEach(ff => ret.push(...f[ff] as string[]));
+      } else if (f.toString().includes('.')) {
+        const parts = f.toString().split('.');
+        const propName = parts.shift();
+        const childPropName = parts.join('.');
+
+        /* istanbul ignore else */
+        if (propName === prop.name) {
+          ret.push(childPropName);
+        }
+      }
+
+      return ret;
+    }, [] as string[]);
+  }
+
   private getChildReferences<T extends AnyEntity<T>>(entities: T[], prop: EntityProperty<T>, refresh: boolean): AnyEntity[] {
     const filtered = this.filterCollections(entities, prop.name, refresh);
     const children: AnyEntity[] = [];

packages/knex/src/AbstractSqlDriver.ts

@@ -2,7 +2,7 @@ import { QueryBuilder as KnexQueryBuilder, Raw, Transaction as KnexTransaction,
 import {
   AnyEntity, Collection, Configuration, Constructor, DatabaseDriver, Dictionary, EntityData, EntityManager, EntityManagerType,
   EntityMetadata, EntityProperty, QueryFlag, FilterQuery, FindOneOptions, FindOptions, IDatabaseDriver, LockMode, Primary,
-  QueryOrderMap, QueryResult, ReferenceType, Transaction, Utils, PopulateOptions, LoadStrategy, CountOptions,
+  QueryOrderMap, QueryResult, ReferenceType, Transaction, Utils, PopulateOptions, LoadStrategy, CountOptions, FieldsMap,
 } from '@mikro-orm/core';
 import { AbstractSqlConnection } from './AbstractSqlConnection';
 import { AbstractSqlPlatform } from './AbstractSqlPlatform';
@@ -39,7 +39,7 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
     const populate = this.autoJoinOneToOneOwner(meta, options.populate as PopulateOptions<T>[], options.fields);
     const joinedProps = this.joinedProps(meta, populate);
     const qb = this.createQueryBuilder<T>(entityName, ctx, !!ctx, false);
-    const fields = this.buildFields(meta, populate, joinedProps, qb, options.fields);
+    const fields = this.buildFields(meta, populate, joinedProps, qb, options.fields as Field<T>[]);
 
     if (Utils.isPrimaryKey(where, meta.compositePK)) {
       where = { [Utils.getPrimaryKeyHash(meta.primaryKeys)]: where } as FilterQuery<T>;
@@ -378,7 +378,7 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
     return this.rethrow(this.updateCollectionDiff<T, O>(meta, coll.property, pks, deleteDiff, insertDiff, ctx));
   }
 
-  async loadFromPivotTable<T extends AnyEntity<T>, O extends AnyEntity<O>>(prop: EntityProperty, owners: Primary<O>[][], where: FilterQuery<T> = {}, orderBy?: QueryOrderMap, ctx?: Transaction): Promise<Dictionary<T[]>> {
+  async loadFromPivotTable<T extends AnyEntity<T>, O extends AnyEntity<O>>(prop: EntityProperty, owners: Primary<O>[][], where: FilterQuery<T> = {}, orderBy?: QueryOrderMap, ctx?: Transaction, options?: FindOptions<T>): Promise<Dictionary<T[]>> {
     const pivotProp2 = this.getPivotInverseProperty(prop);
     const ownerMeta = this.metadata.find(pivotProp2.type)!;
     const targetMeta = this.metadata.find(prop.type)!;
@@ -393,7 +393,8 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
     orderBy = this.getPivotOrderBy(prop, orderBy);
     const qb = this.createQueryBuilder<T>(prop.type, ctx, !!ctx).unsetFlag(QueryFlag.CONVERT_CUSTOM_TYPES);
     const populate = this.autoJoinOneToOneOwner(targetMeta, [{ field: prop.pivotTable }]);
-    qb.select('*').populate(populate).where(where).orderBy(orderBy!);
+    const fields = this.buildFields(targetMeta, (options?.populate ?? []) as PopulateOptions<T>[], [], qb, options?.fields as Field<T>[]);
+    qb.select(fields).populate(populate).where(where).orderBy(orderBy!);
     const items = owners.length ? await this.rethrow(qb.execute('all')) : [];
 
     const map: Dictionary<T[]> = {};
@@ -415,7 +416,7 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
   /**
    * 1:1 owner side needs to be marked for population so QB auto-joins the owner id
    */
-  protected autoJoinOneToOneOwner<T>(meta: EntityMetadata, populate: PopulateOptions<T>[], fields: string[] = []): PopulateOptions<T>[] {
+  protected autoJoinOneToOneOwner<T>(meta: EntityMetadata, populate: PopulateOptions<T>[], fields: (string | FieldsMap)[] = []): PopulateOptions<T>[] {
     if (!this.config.get('autoJoinOneToOneOwner') || fields.length > 0) {
       return populate;
     }
@@ -586,29 +587,38 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
     const lazyProps = meta.props.filter(prop => prop.lazy && !populate.some(p => p.field === prop.name || p.all));
     const hasLazyFormulas = meta.props.some(p => p.lazy && p.formula);
     const hasExplicitFields = !!fields;
+    const ret: Field<T>[] = [];
 
     if (fields) {
-      fields.unshift(...meta.primaryKeys.filter(pk => !fields!.includes(pk)));
+      for (const field of [...fields]) {
+        if (Utils.isPlainObject(field) || field.toString().includes('.')) {
+          continue;
+        }
+
+        ret.push(field);
+      }
+
+      ret.unshift(...meta.primaryKeys.filter(pk => !fields.includes(pk)));
     } else if (joinedProps.length > 0) {
-      fields = this.getFieldsForJoinedLoad(qb, meta, populate);
+      ret.push(...this.getFieldsForJoinedLoad(qb, meta, populate));
     } else if (lazyProps.filter(p => !p.formula).length > 0) {
       const props = meta.props.filter(prop => this.shouldHaveColumn(prop, populate, false));
-      fields = Utils.flatten(props.filter(p => !lazyProps.includes(p)).map(p => p.fieldNames));
+      ret.push(...Utils.flatten(props.filter(p => !lazyProps.includes(p)).map(p => p.fieldNames)));
     } else if (hasLazyFormulas) {
-      fields = ['*'];
+      ret.push('*');
     }
 
-    if (fields && !hasExplicitFields) {
+    if (ret.length > 0 && !hasExplicitFields) {
       meta.props
         .filter(prop => prop.formula && !lazyProps.includes(prop))
         .forEach(prop => {
           const alias = qb.ref(qb.alias).toString();
           const aliased = qb.ref(prop.fieldNames[0]).toString();
-          fields!.push(`${prop.formula!(alias)} as ${aliased}`);
+          ret.push(`${prop.formula!(alias)} as ${aliased}`);
         });
     }
 
-    return fields || ['*'];
+    return ret.length > 0 ? ret : ['*'];
   }
 
 }

packages/mongodb/src/MongoDriver.ts

@@ -1,7 +1,7 @@
 import { ClientSession, ObjectId } from 'mongodb';
 import {
   DatabaseDriver, EntityData, AnyEntity, FilterQuery, EntityMetadata, EntityProperty, Configuration, Utils, ReferenceType, FindOneOptions, FindOptions,
-  QueryResult, Transaction, IDatabaseDriver, EntityManager, EntityManagerType, Dictionary, PopulateOptions, CountOptions,
+  QueryResult, Transaction, IDatabaseDriver, EntityManager, EntityManagerType, Dictionary, PopulateOptions, CountOptions, FieldsMap,
 } from '@mikro-orm/core';
 import { MongoConnection } from './MongoConnection';
 import { MongoPlatform } from './MongoPlatform';
@@ -269,18 +269,35 @@ export class MongoDriver extends DatabaseDriver<MongoConnection> {
     return { _id: id } as FilterQuery<T>;
   }
 
-  private buildFields<T>(entityName: string, populate: PopulateOptions<T>[], fields?: string[]): string[] | undefined {
+  protected buildFields<T extends AnyEntity<T>>(entityName: string, populate: PopulateOptions<T>[], fields?: (string | FieldsMap)[]): string[] | undefined {
     const meta = this.metadata.find(entityName)!;
-    const props = meta.props.filter(prop => this.shouldHaveColumn(prop, populate));
     const lazyProps = meta.props.filter(prop => prop.lazy && !populate.some(p => p.field === prop.name || p.all));
+    const ret: string[] = [];
 
     if (fields) {
-      fields.unshift(...meta.primaryKeys.filter(pk => !fields!.includes(pk)));
-    } else if (lazyProps.length > 0) {
-      fields = Utils.flatten(props.filter(p => !lazyProps.includes(p)).map(p => p.fieldNames));
+      for (const field of fields) {
+        if (Utils.isPlainObject(field) || field.toString().includes('.')) {
+          continue;
+        }
+
+        let prop = meta.properties[field];
+
+        /* istanbul ignore else */
+        if (prop) {
+          prop = prop.serializedPrimaryKey ? meta.getPrimaryProps()[0] : prop;
+          ret.push(prop.fieldNames[0]);
+        } else {
+          ret.push(field);
+        }
+      }
+
+      ret.unshift(...meta.primaryKeys.filter(pk => !fields.includes(pk)));
+    } else if (lazyProps.filter(p => !p.formula).length > 0) {
+      const props = meta.props.filter(prop => this.shouldHaveColumn(prop, populate));
+      ret.push(...Utils.flatten(props.filter(p => !lazyProps.includes(p)).map(p => p.fieldNames)));
     }
 
-    return fields;
+    return ret.length > 0 ? ret : undefined;
   }
 
   shouldHaveColumn<T>(prop: EntityProperty<T>, populate: PopulateOptions<T>[]): boolean {

tests/EntityManager.mongo.test.ts

@@ -1657,18 +1657,6 @@ describe('EntityManagerMongo', () => {
     expect(b2.title).toBe('test 2');
   });
 
-  test('partial selects', async () => {
-    const author = new Author('Jon Snow', 'snow@wall.st');
-    author.born = new Date();
-    await orm.em.persistAndFlush(author);
-    orm.em.clear();
-
-    const a = (await orm.em.findOne(Author, author, { fields: ['name'] }))!;
-    expect(a.name).toBe('Jon Snow');
-    expect(a.email).toBeUndefined();
-    expect(a.born).toBeUndefined();
-  });
-
   test(`populating inverse side of 1:1 also back-links inverse side's owner (both eager)`, async () => {
     const bar = FooBar.create('fb');
     bar.baz = FooBaz.create('fz');
@@ -1898,6 +1886,8 @@ describe('EntityManagerMongo', () => {
     await orm.em.flush();
     orm.em.clear();
 
+    const tags = await orm.em.find(BookTag, {}, ['books']);
+    console.log(tags);
     let tag = await orm.em.findOneOrFail(BookTag, tag1.id, ['books']);
     const err = 'You cannot modify inverse side of M:N collection BookTag.books when the owning side is not initialized. Consider working with the owning side instead (Book.tags).';
     expect(() => tag.books.add(orm.em.getReference(Book, book4.id))).toThrowError(err);