feat(core): add support for virtual entities (#3351)

docs/docs/virtual-entities.md

@@ -0,0 +1,242 @@
+---
+title: Virtual Entities
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+Virtual entities don't represent any database table. Instead, they dynamically resolve to an SQL query (or an aggregation in mongo), allowing to map any kind of results onto an entity. Such entities are mean for read purposes, they don't have a primary key and therefore cannot be tracked for changes. In a sense they are similar to (currently unsupported) database views. 
+
+To define a virtual entity, provide an `expression`, either as a string (SQL query):
+
+> We need to use the virtual column names based on current naming strategy. Note the `authorName` property being represented as `author_name` column.
+
+<Tabs
+  groupId="entity-def"
+  defaultValue="reflect-metadata"
+  values={[
+    {label: 'reflect-metadata', value: 'reflect-metadata'},
+    {label: 'ts-morph', value: 'ts-morph'},
+    {label: 'EntitySchema', value: 'entity-schema'},
+  ]
+  }>
+  <TabItem value="reflect-metadata">
+
+```ts title="./entities/BookWithAuthor.ts"
+@Entity({
+  expression: 'select name, age, ' +
+    '(select count(*) from book b where b.author_id = a.id) as total_books, ' +
+    '(select group_concat(distinct t.name) from book b ' +
+      'join tags_ordered bt on bt.book_id = b.id ' +
+      'join book_tag t on t.id = bt.book_tag_id ' +
+      'where b.author_id = a.id ' +
+      'group by b.author_id) as used_tags ' +
+    'from author a group by a.id',
+})
+export class BookWithAuthor {
+
+  @Property()
+  title!: string;
+
+  @Property()
+  authorName!: string;
+
+  @Property()
+  tags!: string[];
+
+}
+```
+
+  </TabItem>
+  <TabItem value="ts-morph">
+
+```ts title="./entities/BookWithAuthor.ts"
+@Entity({
+  expression: 'select name, age, ' +
+    '(select count(*) from book b where b.author_id = a.id) as total_books, ' +
+    '(select group_concat(distinct t.name) from book b ' +
+      'join tags_ordered bt on bt.book_id = b.id ' +
+      'join book_tag t on t.id = bt.book_tag_id ' +
+      'where b.author_id = a.id ' +
+      'group by b.author_id) as used_tags ' +
+    'from author a group by a.id',
+})
+export class BookWithAuthor {
+
+  @Property()
+  title!: string;
+
+  @Property()
+  authorName!: string;
+
+  @Property()
+  tags!: string[];
+
+}
+```
+
+  </TabItem>
+  <TabItem value="entity-schema">
+
+```ts title="./entities/BookWithAuthor.ts"
+export interface IBookWithAuthor{
+  title: string;
+  authorName: string;
+  tags: string[];
+}
+
+export const BookWithAuthor = new EntitySchema<IBookWithAuthor>({ 
+  name: 'BookWithAuthor',
+  expression: 'select name, age, ' +
+    '(select count(*) from book b where b.author_id = a.id) as total_books, ' +
+    '(select group_concat(distinct t.name) from book b ' +
+    'join tags_ordered bt on bt.book_id = b.id ' +
+    'join book_tag t on t.id = bt.book_tag_id ' +
+    'where b.author_id = a.id ' +
+    'group by b.author_id) as used_tags ' +
+    'from author a group by a.id',
+  properties: {
+    title: { type: 'string' },
+    authorName: { type: 'string' },
+    tags: { type: 'string[]' },
+  },
+});
+```
+
+  </TabItem>
+</Tabs>
+
+Or as a callback:
+
+<Tabs
+  groupId="entity-def"
+  defaultValue="reflect-metadata"
+  values={[
+    {label: 'reflect-metadata', value: 'reflect-metadata'},
+    {label: 'ts-morph', value: 'ts-morph'},
+    {label: 'EntitySchema', value: 'entity-schema'},
+  ]
+  }>
+  <TabItem value="reflect-metadata">
+
+```ts title="./entities/BookWithAuthor.ts"
+@Entity({
+  expression: (em: EntityManager) => {
+    return em.createQueryBuilder(Book, 'b')
+      .select(['b.title', 'a.name as author_name', 'group_concat(t.name) as tags'])
+      .join('b.author', 'a')
+      .join('b.tags', 't')
+      .groupBy('b.id');
+  },
+})
+export class BookWithAuthor {
+
+  @Property()
+  title!: string;
+
+  @Property()
+  authorName!: string;
+
+  @Property()
+  tags!: string[];
+
+}
+```
+
+  </TabItem>
+  <TabItem value="ts-morph">
+
+```ts title="./entities/BookWithAuthor.ts"
+@Entity({
+  expression: (em: EntityManager) => {
+    return em.createQueryBuilder(Book, 'b')
+      .select(['b.title', 'a.name as author_name', 'group_concat(t.name) as tags'])
+      .join('b.author', 'a')
+      .join('b.tags', 't')
+      .groupBy('b.id');
+  },
+})
+export class BookWithAuthor {
+
+  @Property()
+  title!: string;
+
+  @Property()
+  authorName!: string;
+
+  @Property()
+  tags!: string[];
+
+}
+```
+
+  </TabItem>
+  <TabItem value="entity-schema">
+
+```ts title="./entities/BookWithAuthor.ts"
+export interface IBookWithAuthor{
+  title: string;
+  authorName: string;
+  tags: string[];
+}
+
+export const BookWithAuthor = new EntitySchema<IBookWithAuthor>({
+  name: 'BookWithAuthor',
+  expression: (em: EntityManager) => {
+    return em.createQueryBuilder(Book, 'b')
+      .select(['b.title', 'a.name as author_name', 'group_concat(t.name) as tags'])
+      .join('b.author', 'a')
+      .join('b.tags', 't')
+      .groupBy('b.id');
+  },
+  properties: {
+    title: { type: 'string' },
+    authorName: { type: 'string' },
+    tags: { type: 'string[]' },
+  },
+});
+```
+
+  </TabItem>
+</Tabs>
+
+In MongoDB, we can use aggregations, although it is not very ergonomic due to their nature. Following example is a rough equivalent of the previous SQL ones.
+
+> The `where` query as well as the options like `orderBy`, `limit` and `offset` needs to be explicitly handled in your pipeline.
+
+```ts
+@Entity({
+  expression: (em: EntityManager, where, options) => {
+    const $sort = { ...options.orderBy } as Dictionary;
+    $sort._id = 1;
+    const pipeline: Dictionary[] = [
+      { $project: { _id: 0, title: 1, author: 1 } },
+      { $sort },
+      { $match: where ?? {} },
+      { $lookup: { from: 'author', localField: 'author', foreignField: '_id', as: 'author', pipeline: [{ $project: { name: 1 } }] } },
+      { $unwind: '$author' },
+      { $set: { authorName: '$author.name' } },
+      { $unset: ['author'] },
+    ];
+
+    if (options.offset != null) {
+      pipeline.push({ $skip: options.offset });
+    }
+
+    if (options.limit != null) {
+      pipeline.push({ $limit: options.limit });
+    }
+
+    return em.aggregate(Book, pipeline);
+  },
+})
+export class BookWithAuthor {
+
+  @Property()
+  title!: string;
+
+  @Property()
+  authorName!: string;
+
+}
+```

docs/sidebars.js

@@ -54,6 +54,7 @@ module.exports = {
         'events',
         'composite-keys',
         'custom-types',
+        'virtual-entities',
         'embeddables',
         'entity-schema',
         'json-properties',

packages/core/src/EntityManager.ts

@@ -132,6 +132,7 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
       return [];
     }
 
+    const meta = this.metadata.get(entityName);
     const ret: T[] = [];
 
     for (const data of results) {
@@ -141,10 +142,21 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
         schema: options.schema,
         convertCustomTypes: true,
       }) as T;
-      em.unitOfWork.registerManaged(entity, data, { refresh: options.refresh, loaded: true });
+
+      if (!meta.virtual) {
+        em.unitOfWork.registerManaged(entity, data, { refresh: options.refresh, loaded: true });
+      }
+
       ret.push(entity);
     }
 
+    if (meta.virtual) {
+      await em.unitOfWork.dispatchOnLoadEvent();
+      await em.storeCache(options.cache, cached!, () => ret.map(e => e.__helper!.toPOJO()));
+
+      return ret as Loaded<T, P>[];
+    }
+
     const unique = Utils.unique(ret);
     await em.entityLoader.populate<T, P>(entityName, unique, populate, {
       ...options as Dictionary,
@@ -385,8 +397,12 @@ export class EntityManager<D extends IDatabaseDriver = IDatabaseDriver> {
       schema: options.schema,
       convertCustomTypes: true,
     });
-    em.unitOfWork.registerManaged(entity, data, { refresh: options.refresh, loaded: true });
-    await em.lockAndPopulate(entityName, entity, where, options);
+
+    if (!meta.virtual) {
+      em.unitOfWork.registerManaged(entity, data, { refresh: options.refresh, loaded: true });
+      await em.lockAndPopulate(entityName, entity, where, options);
+    }
+
     await em.unitOfWork.dispatchOnLoadEvent();
     await em.storeCache(options.cache, cached!, () => entity!.__helper!.toPOJO());
 

packages/core/src/decorators/Entity.ts

@@ -1,6 +1,7 @@
 import { MetadataStorage } from '../metadata';
 import { Utils } from '../utils';
-import type { Constructor, Dictionary } from '../typings';
+import type { Constructor, Dictionary, FilterQuery } from '../typings';
+import type { FindOptions } from '../drivers/IDatabaseDriver';
 
 export function Entity(options: EntityOptions<any> = {}) {
   return function <T>(target: T & Dictionary) {
@@ -26,5 +27,9 @@ export type EntityOptions<T> = {
   comment?: string;
   abstract?: boolean;
   readonly?: boolean;
+  virtual?: boolean;
+  // we need to use `em: any` here otherwise an expression would not be assignable with more narrow type like `SqlEntityManager`
+  // also return type is unknown as it can be either QB instance (which we cannot type here) or array of POJOs (e.g. for mongodb)
+  expression?: string | ((em: any, where: FilterQuery<T>, options: FindOptions<T, any>) => object);
   customRepository?: () => Constructor;
 };

packages/core/src/drivers/DatabaseDriver.ts

@@ -49,6 +49,11 @@ export abstract class DatabaseDriver<C extends Connection> implements IDatabaseD
     return new EntityManager(this.config, this, this.metadata, useContext) as unknown as EntityManager<D>;
   }
 
+  /* istanbul ignore next */
+  async findVirtual<T>(entityName: string, where: FilterQuery<T>, options: FindOptions<T, any>): Promise<EntityData<T>[]> {
+    throw new Error(`Virtual entities are not supported by ${this.constructor.name} driver.`);
+  }
+
   async aggregate(entityName: string, pipeline: any[]): Promise<any[]> {
     throw new Error(`Aggregations are not supported by ${this.constructor.name} driver`);
   }

packages/core/src/drivers/IDatabaseDriver.ts

@@ -38,6 +38,8 @@ export interface IDatabaseDriver<C extends Connection = Connection> {
    */
   findOne<T extends AnyEntity<T>, P extends string = never>(entityName: string, where: FilterQuery<T>, options?: FindOneOptions<T, P>): Promise<EntityData<T> | null>;
 
+  findVirtual<T>(entityName: string, where: FilterQuery<T>, options: FindOptions<T, any>): Promise<EntityData<T>[]>;
+
   nativeInsert<T extends AnyEntity<T>>(entityName: string, data: EntityDictionary<T>, options?: NativeInsertUpdateOptions<T>): Promise<QueryResult<T>>;
 
   nativeInsertMany<T extends AnyEntity<T>>(entityName: string, data: EntityDictionary<T>[], options?: NativeInsertUpdateManyOptions<T>): Promise<QueryResult<T>>;

packages/core/src/entity/EntityFactory.ts

@@ -36,6 +36,14 @@ export class EntityFactory {
     entityName = Utils.className(entityName);
     const meta = this.metadata.get(entityName);
 
+    if (meta.virtual) {
+      data = { ...data };
+      const entity = this.createEntity<T>(data, meta, options);
+      this.hydrate(entity, meta, data, options);
+
+      return entity as New<T, P>;
+    }
+
     if (this.platform.usesDifferentSerializedPrimaryKey()) {
       meta.primaryKeys.forEach(pk => this.denormalizePrimaryKey(data, pk, meta.properties[pk]));
     }
@@ -161,7 +169,7 @@ export class EntityFactory {
   }
 
   private createEntity<T extends AnyEntity<T>>(data: EntityData<T>, meta: EntityMetadata<T>, options: FactoryOptions): T {
-    if (options.newEntity || meta.forceConstructor) {
+    if (options.newEntity || meta.forceConstructor || meta.virtual) {
       if (!meta.class) {
         throw new Error(`Cannot create entity ${meta.className}, class prototype is unknown`);
       }
@@ -173,6 +181,11 @@ export class EntityFactory {
 
       // creates new instance via constructor as this is the new entity
       const entity = new Entity(...params);
+
+      if (meta.virtual) {
+        return entity;
+      }
+
       entity.__helper!.__schema = this.driver.getSchemaName(meta, options);
 
       if (!options.newEntity) {
@@ -211,7 +224,7 @@ export class EntityFactory {
     } else {
       this.hydrator.hydrateReference(entity, meta, data, this, options.convertCustomTypes, this.driver.getSchemaName(meta, options));
     }
-    Object.keys(data).forEach(key => entity.__helper!.__loadedProperties.add(key));
+    Object.keys(data).forEach(key => entity.__helper?.__loadedProperties.add(key));
   }
 
   private findEntity<T>(data: EntityData<T>, meta: EntityMetadata<T>, options: FactoryOptions): T | undefined {

packages/core/src/hydration/ObjectHydrator.ts

@@ -316,7 +316,7 @@ export class ObjectHydrator extends Hydrator {
   }
 
   private createCollectionItemMapper<T>(prop: EntityProperty): string[] {
-    const meta = this.metadata.find(prop.type)!;
+    const meta = this.metadata.get(prop.type);
     const lines: string[] = [];
 
     lines.push(`  const createCollectionItem_${this.safeKey(prop.name)} = value => {`);

packages/core/src/metadata/MetadataStorage.ts

@@ -103,7 +103,7 @@ export class MetadataStorage {
 
   decorate(em: EntityManager): void {
     Object.values(this.metadata)
-      .filter(meta => meta.prototype && !meta.prototype.__meta)
+      .filter(meta => meta.prototype && !meta.prototype.__meta && !meta.virtual)
       .forEach(meta => EntityHelper.decorate(meta, em));
   }