feat(core): allow populating collections with references (#4776)

Sometimes we might want to know only what items are part of a
collection, and we don't care about the values of those items. For this,
we can populate the collection only with references:

```ts
const book1 = await em.findOne(Book, 1, { populate: ['tags:ref'] });
console.log(book1.tags.isInitialized()); // true
console.log(wrap(book1.tags[0]).isInitialized()); // false

// or alternatively use `init({ ref: true })`
const book2 = await em.findOne(Book, 1);
await book2.tags.init({ ref: true });
console.log(book2.tags.isInitialized()); // true
console.log(wrap(book2.tags[0]).isInitialized()); // false
```

Closes #1158

docs/docs/collections.md

@@ -222,6 +222,22 @@ To preserve fixed order of collections, we can use `fixedOrder: true` attribute,
 
 We can also specify default ordering via `orderBy: { ... }` attribute. This will be used when we fully populate the collection including its items, as it orders by the referenced entity properties instead of pivot table columns (which `fixedOrderColumn` is). On the other hand, `fixedOrder` is used to maintain the insert order of items instead of ordering by some property.
 
+## Populating references
+
+Sometimes we might want to know only what items are part of a collection, and we don't care about the values of those items. For this, we can populate the collection only with references:
+
+```ts
+const book1 = await em.findOne(Book, 1, { populate: ['tags:ref'] });
+console.log(book1.tags.isInitialized()); // true
+console.log(wrap(book1.tags[0]).isInitialized()); // false
+
+// or alternatively use `init({ ref: true })`
+const book2 = await em.findOne(Book, 1);
+await book2.tags.init({ ref: true });
+console.log(book2.tags.isInitialized()); // true
+console.log(wrap(book2.tags[0]).isInitialized()); // false
+```
+
 ## Propagation of Collection's add() and remove() operations
 
 When we use one of `Collection.add()` method, the item is added to given collection, and this action is also propagated to its counterpart.

packages/core/src/EntityManager.ts

@@ -384,7 +384,8 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
 
     if (options.populate) {
       for (const hint of (options.populate as unknown as PopulateOptions<Entity>[])) {
-        const prop = meta.properties[hint.field];
+        const field = hint.field.split(':')[0] as EntityKey<Entity>;
+        const prop = meta.properties[field];
         const joined = (hint.strategy || prop.strategy || this.config.get('loadStrategy')) === LoadStrategy.JOINED && prop.kind !== ReferenceKind.SCALAR;
 
         if (!joined) {
@@ -395,14 +396,14 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
         const where2 = await this.applyJoinedFilters<Entity>(prop.targetMeta!, {} as ObjectQuery<Entity>, { ...options, populate: hint.children as any, populateWhere: PopulateHint.ALL });
 
         if (Utils.hasObjectKeys(where!)) {
-          ret[hint.field] = ret[hint.field] ? { $and: [where, ret[hint.field]] } : where as any;
+          ret[field] = ret[field] ? { $and: [where, ret[field]] } : where as any;
         }
 
         if (Utils.hasObjectKeys(where2)) {
-          if (ret[hint.field]) {
-            Utils.merge(ret[hint.field], where2);
+          if (ret[field]) {
+            Utils.merge(ret[field], where2);
           } else {
-            ret[hint.field] = where2 as any;
+            ret[field] = where2 as any;
           }
         }
       }
@@ -1593,13 +1594,18 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
    */
   canPopulate<Entity extends object>(entityName: EntityName<Entity>, property: string): boolean {
     entityName = Utils.className(entityName);
-    const [p, ...parts] = property.split('.');
+    // eslint-disable-next-line prefer-const
+    let [p, ...parts] = property.split('.');
     const meta = this.metadata.find(entityName);
 
     if (!meta) {
       return true;
     }
 
+    if (p.includes(':')) {
+      p = p.split(':', 2)[0];
+    }
+
     const ret = p in meta.properties;
 
     if (!ret) {
@@ -1839,6 +1845,10 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
     Hint extends string = never,
     Fields extends string = never,
   >(entityName: string, options: Pick<FindOptions<Entity, Hint, Fields>, 'populate' | 'strategy' | 'fields'>): PopulateOptions<Entity>[] {
+    if (options.populate === false) {
+      return [];
+    }
+
     // infer populate hint if only `fields` are available
     if (!options.populate && options.fields) {
       const meta = this.metadata.find(entityName)!;

packages/core/src/drivers/DatabaseDriver.ts

@@ -88,7 +88,7 @@ export abstract class DatabaseDriver<C extends Connection> implements IDatabaseD
     throw new Error(`Aggregations are not supported by ${this.constructor.name} driver`);
   }
 
-  async loadFromPivotTable<T extends object, O extends object>(prop: EntityProperty, owners: Primary<O>[][], where?: FilterQuery<any>, orderBy?: OrderDefinition<T>, ctx?: Transaction, options?: FindOptions<T, any, any>): Promise<Dictionary<T[]>> {
+  async loadFromPivotTable<T extends object, O extends object>(prop: EntityProperty, owners: Primary<O>[][], where?: FilterQuery<any>, orderBy?: OrderDefinition<T>, ctx?: Transaction, options?: FindOptions<T, any, any>, pivotJoin?: boolean): Promise<Dictionary<T[]>> {
     throw new Error(`${this.constructor.name} does not use pivot tables`);
   }
 
@@ -318,16 +318,6 @@ export abstract class DatabaseDriver<C extends Connection> implements IDatabaseD
     return meta ? Utils.flatten(meta.getPrimaryProps().map(pk => pk.fieldNames)) : [this.config.getNamingStrategy().referenceColumnName()];
   }
 
-  protected getPivotInverseProperty(prop: EntityProperty): EntityProperty {
-    const pivotMeta = this.metadata.find(prop.pivotEntity)!;
-
-    if (prop.owner) {
-      return pivotMeta.relations[0];
-    }
-
-    return pivotMeta.relations[1];
-  }
-
   protected createReplicas(cb: (c: ConnectionOptions) => C): C[] {
     const replicas = this.config.get('replicas', [])!;
     const ret: C[] = [];

packages/core/src/drivers/IDatabaseDriver.ts

@@ -62,7 +62,7 @@ export interface IDatabaseDriver<C extends Connection = Connection> {
   /**
    * When driver uses pivot tables for M:N, this method will load identifiers for given collections from them
    */
-  loadFromPivotTable<T extends object, O extends object>(prop: EntityProperty, owners: Primary<O>[][], where?: FilterQuery<T>, orderBy?: OrderDefinition<T>, ctx?: Transaction, options?: FindOptions<T, any, any>): Promise<Dictionary<T[]>>;
+  loadFromPivotTable<T extends object, O extends object>(prop: EntityProperty, owners: Primary<O>[][], where?: FilterQuery<T>, orderBy?: OrderDefinition<T>, ctx?: Transaction, options?: FindOptions<T, any, any>, pivotJoin?: boolean): Promise<Dictionary<T[]>>;
 
   getPlatform(): Platform;
 

packages/core/src/entity/Collection.ts

@@ -1,23 +1,23 @@
 import type {
   AnyEntity,
+  ConnectionType,
+  Dictionary,
   EntityData,
   EntityDTO,
+  EntityKey,
   EntityMetadata,
+  EntityValue,
+  FilterKey,
   FilterQuery,
   Loaded,
   LoadedCollection,
   Populate,
   Primary,
-  ConnectionType,
-  Dictionary,
-  FilterKey,
-  EntityKey,
-  EntityValue,
 } from '../typings';
 import { ArrayCollection } from './ArrayCollection';
 import { Utils } from '../utils/Utils';
 import { ValidationError } from '../errors';
-import { ReferenceKind, type LockMode, type QueryOrderMap } from '../enums';
+import { type LockMode, type QueryOrderMap, ReferenceKind } from '../enums';
 import { Reference } from './Reference';
 import type { Transaction } from '../connections/Connection';
 import type { FindOptions } from '../drivers/IDatabaseDriver';
@@ -325,14 +325,15 @@ export class Collection<T extends object, O extends object = object> extends Arr
 
     const populate = Array.isArray(options.populate)
       ? options.populate.map(f => `${this.property.name}.${f}`)
-      : [this.property.name];
+      : [`${this.property.name}${options.ref ? ':ref' : ''}`];
+    const schema = this.property.targetMeta!.schema === '*'
+      ? helper(this.owner).__schema
+      : undefined;
     await em.populate(this.owner, populate, {
       ...options,
       refresh: true,
       connectionType: options.connectionType,
-      schema: this.property.targetMeta!.schema === '*'
-        ? helper(this.owner).__schema
-        : this.property.targetMeta!.schema,
+      schema,
       where: { [this.property.name]: options.where },
       orderBy: { [this.property.name]: options.orderBy },
     });
@@ -500,6 +501,7 @@ Object.defineProperties(Collection.prototype, {
 
 export interface InitOptions<T, P extends string = never> {
   populate?: Populate<T, P>;
+  ref?: boolean; // populate only references, works only with M:N collections that use pivot table
   orderBy?: QueryOrderMap<T> | QueryOrderMap<T>[];
   where?: FilterQuery<T>;
   lockMode?: Exclude<LockMode, LockMode.OPTIMISTIC>;

packages/core/src/entity/EntityLoader.ts

@@ -1,21 +1,21 @@
 import type {
   AnyEntity,
+  ConnectionType,
   Dictionary,
+  EntityKey,
   EntityMetadata,
   EntityProperty,
+  FilterKey,
   FilterQuery,
   PopulateOptions,
   Primary,
-  ConnectionType,
-  EntityKey,
-  FilterKey,
 } from '../typings';
 import type { EntityManager } from '../EntityManager';
 import { QueryHelper } from '../utils/QueryHelper';
 import { Utils } from '../utils/Utils';
 import { ValidationError } from '../errors';
 import type { Collection } from './Collection';
-import { LoadStrategy, ReferenceKind, type LockMode, type PopulateHint, type QueryOrderMap } from '../enums';
+import { LoadStrategy, type LockMode, type PopulateHint, type QueryOrderMap, ReferenceKind } from '../enums';
 import { Reference, type ScalarReference } from './Reference';
 import type { EntityField, FindOptions, IDatabaseDriver } from '../drivers/IDatabaseDriver';
 import type { MetadataStorage } from '../metadata/MetadataStorage';
@@ -176,7 +176,7 @@ export class EntityLoader {
    * preload everything in one call (this will update already existing references in IM)
    */
   private async populateMany<Entity extends object>(entityName: string, entities: Entity[], populate: PopulateOptions<Entity>, options: Required<EntityLoaderOptions<Entity>>): Promise<AnyEntity[]> {
-    const field = populate.field as EntityKey<Entity>;
+    const [field, ref] = populate.field.split(':', 2) as [EntityKey<Entity>, string | undefined];
     const meta = this.metadata.find<Entity>(entityName)!;
     const prop = meta.properties[field];
 
@@ -211,11 +211,11 @@ export class EntityLoader {
       .flatMap(orderBy => orderBy[prop.name]);
 
     if (prop.kind === ReferenceKind.MANY_TO_MANY && this.driver.getPlatform().usesPivotTable()) {
-      return this.findChildrenFromPivotTable<Entity>(filtered, prop, options, innerOrderBy as QueryOrderMap<Entity>[], populate);
+      return this.findChildrenFromPivotTable<Entity>(filtered, prop, options, innerOrderBy as QueryOrderMap<Entity>[], populate, !!ref);
     }
 
     const where = await this.extractChildCondition(options, prop);
-    const data = await this.findChildren<Entity>(entities, prop, populate, { ...options, where, orderBy: innerOrderBy! });
+    const data = await this.findChildren<Entity>(entities, prop, populate, { ...options, where, orderBy: innerOrderBy! }, !!ref);
     this.initializeCollections<Entity>(filtered, prop, field, data, innerOrderBy.length > 0);
 
     return data;
@@ -286,9 +286,9 @@ export class EntityLoader {
     }
   }
 
-  private async findChildren<Entity extends object>(entities: Entity[], prop: EntityProperty<Entity>, populate: PopulateOptions<Entity>, options: Required<EntityLoaderOptions<Entity>>): Promise<AnyEntity[]> {
+  private async findChildren<Entity extends object>(entities: Entity[], prop: EntityProperty<Entity>, populate: PopulateOptions<Entity>, options: Required<EntityLoaderOptions<Entity>>, ref: boolean): Promise<AnyEntity[]> {
     const children = this.getChildReferences<Entity>(entities, prop, options);
-    const meta = this.metadata.find(prop.type)!;
+    const meta = prop.targetMeta!;
     let fk = Utils.getPrimaryKeyHash(meta.primaryKeys);
     let schema: string | undefined = options.schema;
 
@@ -312,17 +312,27 @@ export class EntityLoader {
 
     const ids = Utils.unique(children.map(e => e.__helper.getPrimaryKey()));
     const where = this.mergePrimaryCondition<Entity>(ids, fk as FilterKey<Entity>, options, meta, this.metadata, this.driver.getPlatform());
-    const fields = this.buildFields(options.fields, prop) as any;
+    const fields = this.buildFields(options.fields, prop, ref) as any;
     const { refresh, filters, convertCustomTypes, lockMode, strategy, populateWhere, connectionType, logging } = options;
 
-    return this.em.find(prop.type, where, {
+    const items = await this.em.find(prop.type, where, {
       refresh, filters, convertCustomTypes, lockMode, populateWhere, logging,
       orderBy: [...Utils.asArray(options.orderBy), ...Utils.asArray(prop.orderBy)] as QueryOrderMap<Entity>[],
       populate: populate.children as never ?? populate.all ?? [],
       strategy, fields, schema, connectionType,
       // @ts-ignore not a public option, will be propagated to the populate call
       visited: options.visited,
     });
+
+    for (const item of items) {
+      if (ref && !helper(item).__onLoadFired) {
+        helper(item).__initialized = false;
+        // eslint-disable-next-line dot-notation
+        this.em.getUnitOfWork()['loadedEntities'].delete(item);
+      }
+    }
+
+    return items;
   }
 
   private mergePrimaryCondition<Entity>(ids: Entity[], pk: FilterKey<Entity>, options: EntityLoaderOptions<Entity>, meta: EntityMetadata, metadata: MetadataStorage, platform: Platform): FilterQuery<Entity> {
@@ -334,7 +344,8 @@ export class EntityLoader {
   }
 
   private async populateField<Entity extends object>(entityName: string, entities: Entity[], populate: PopulateOptions<Entity>, options: Required<EntityLoaderOptions<Entity>>): Promise<void> {
-    const prop = this.metadata.find(entityName)!.properties[populate.field] as EntityProperty<Entity>;
+    const field = populate.field.split(':')[0] as EntityKey<Entity>;
+    const prop = this.metadata.find(entityName)!.properties[field] as EntityProperty<Entity>;
 
     if (prop.kind === ReferenceKind.SCALAR && !prop.lazy) {
       return;
@@ -349,7 +360,7 @@ export class EntityLoader {
     const children: Entity[] = [];
 
     for (const entity of entities) {
-      const ref = entity[populate.field] as unknown;
+      const ref = entity[field] as unknown;
 
       if (Utils.isEntity<Entity>(ref)) {
         children.push(ref);
@@ -391,7 +402,7 @@ export class EntityLoader {
     });
   }
 
-  private async findChildrenFromPivotTable<Entity extends object>(filtered: Entity[], prop: EntityProperty<Entity>, options: Required<EntityLoaderOptions<Entity>>, orderBy?: QueryOrderMap<Entity>[], populate?: PopulateOptions<Entity>): Promise<AnyEntity[]> {
+  private async findChildrenFromPivotTable<Entity extends object>(filtered: Entity[], prop: EntityProperty<Entity>, options: Required<EntityLoaderOptions<Entity>>, orderBy?: QueryOrderMap<Entity>[], populate?: PopulateOptions<Entity>, pivotJoin?: boolean): Promise<AnyEntity[]> {
     const ids = (filtered as AnyEntity[]).map(e => e.__helper!.__primaryKeys);
     const refresh = options.refresh;
     const where = await this.extractChildCondition(options, prop, true);
@@ -406,11 +417,18 @@ export class EntityLoader {
       ids.forEach((id, idx) => ids[idx] = QueryHelper.processCustomType<Entity>(prop, id as FilterQuery<Entity>, this.driver.getPlatform()) as Primary<Entity>[]);
     }
 
-    const map = await this.driver.loadFromPivotTable<any, any>(prop, ids, where, orderBy, this.em.getTransactionContext(), options2);
+    const map = await this.driver.loadFromPivotTable<any, any>(prop, ids, where, orderBy, this.em.getTransactionContext(), options2, pivotJoin);
     const children: AnyEntity[] = [];
 
     for (const entity of (filtered as AnyEntity[])) {
       const items = map[entity.__helper!.getSerializedPrimaryKey()].map(item => {
+        if (pivotJoin) {
+          return this.em.getReference(prop.type, item, {
+            convertCustomTypes: true,
+            schema: options.schema ?? this.em.config.get('schema'),
+          });
+        }
+
         const entity = this.em.getEntityFactory().create(prop.type, item, {
           refresh,
           merge: true,
@@ -468,7 +486,11 @@ export class EntityLoader {
     return subCond;
   }
 
-  private buildFields<Entity, Hint extends string>(fields: readonly EntityField<Entity, Hint>[] = [], prop: EntityProperty<Entity>): readonly EntityField<Entity>[] | undefined {
+  private buildFields<Entity>(fields: readonly EntityField<Entity>[] = [], prop: EntityProperty<Entity>, ref?: boolean): readonly EntityField<Entity>[] | undefined {
+    if (ref) {
+      fields = prop.targetMeta!.primaryKeys.map(targetPkName => `${prop.name}.${targetPkName}`) as EntityField<Entity>[];
+    }
+
     const ret = fields.reduce((ret, f) => {
       if (Utils.isPlainObject(f)) {
         Utils.keys(f)
@@ -493,10 +515,10 @@ export class EntityLoader {
     }
 
     // we need to automatically select the FKs too, e.g. for 1:m relations to be able to wire them with the items
-    if (prop.kind === ReferenceKind.ONE_TO_MANY) {
+    if (prop.kind === ReferenceKind.ONE_TO_MANY || prop.kind === ReferenceKind.MANY_TO_MANY) {
       const owner = prop.targetMeta!.properties[prop.mappedBy] as EntityProperty<Entity>;
 
-      if (!ret.includes(owner.name)) {
+      if (owner && !ret.includes(owner.name)) {
         ret.push(owner.name);
       }
     }
@@ -604,7 +626,12 @@ export class EntityLoader {
     const ret: PopulateOptions<Entity>[] = prefix === '' ? [...populate] : [];
 
     meta.relations
-      .filter(prop => prop.eager || populate.some(p => p.field === prop.name))
+      .filter(prop => {
+        const eager = prop.eager && !populate.some(p => p.field === `${prop.name}:ref`);
+        const populated = populate.some(p => p.field === prop.name);
+
+        return eager || populated;
+      })
       .forEach(prop => {
         const field = this.getRelationName(meta, prop);
         const prefixed = prefix ? `${prefix}.${field}` as EntityKey<Entity> : field;

packages/core/src/metadata/MetadataDiscovery.ts

@@ -745,6 +745,7 @@ export class MetadataDiscovery {
     }
 
     this.initColumnType(ret);
+    this.initRelation(ret);
 
     return ret;
   }

packages/core/src/typings.ts

@@ -39,6 +39,8 @@ export type Compute<T> = { [K in keyof T]: T[K] } & {};
 export type ExcludeFunctions<T, K extends keyof T> = T[K] extends Function ? never : (K extends symbol ? never : K);
 export type Cast<T, R> = T extends R ? T : R;
 export type IsUnknown<T> = T extends unknown ? unknown extends T ? true : never : never;
+export type IsAny<T> = 0 extends (1 & T) ? true : false;
+export type IsNever<T> = [T] extends [never] ? true : false;
 export type NoInfer<T> = [T][T extends any ? 0 : never];
 
 export type DeepPartial<T> = T & {
@@ -800,6 +802,17 @@ type GetStringKey<T, K extends StringKeys<T, string>, E extends string> = K exte
 // limit depth of the recursion to 5 (inspired by https://www.angularfix.com/2022/01/why-am-i-getting-instantiation-is.html)
 type Prev = [never, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
 
+// for pivot joining via populate hint, e.g. `tags:ref`
+type CollectionKeys<T> = T extends object
+  ? {
+    [K in keyof T]: T[K] extends Collection<any>
+      ? IsAny<T[K]> extends true
+        ? never
+        : K & string
+      : never
+  }[keyof T] & {}
+  : never;
+
 export type AutoPath<O, P extends string, E extends string = never, D extends Prev[number] = 5> =
   [D] extends [never] ? any :
   P extends any ?
@@ -811,7 +824,7 @@ export type AutoPath<O, P extends string, E extends string = never, D extends Pr
           : never
         : Q extends StringKeys<O, E>
           ? (Defined<GetStringKey<O, Q, E>> extends unknown ? Exclude<P, `${string}.`> : never) | (StringKeys<Defined<GetStringKey<O, Q, E>>, E> extends never ? never : `${Q}.`)
-          : StringKeys<O, E>
+          : StringKeys<O, E> | `${CollectionKeys<O>}:ref`
       : never
     : never;