fix(core): support partial loading of inlined embeddables

Closes #3365

packages/core/src/entity/EntityTransformer.ts

@@ -4,6 +4,7 @@ import type { Reference } from './Reference';
 import { helper, wrap } from './wrap';
 import type { Platform } from '../platforms';
 import { Utils } from '../utils/Utils';
+import { ReferenceType } from '../enums';
 
 /**
  * Helper that allows to keep track of where we are currently at when serializing complex entity graph with cycles.
@@ -212,6 +213,16 @@ export class EntityTransformer {
       return EntityTransformer.processEntity(prop, entity, wrapped.__platform, raw);
     }
 
+    if (property.reference === ReferenceType.EMBEDDED) {
+      if (Array.isArray(entity[prop])) {
+        return (entity[prop] as object[]).map(item => helper(item).toPOJO()) as T[keyof T];
+      }
+
+      if (Utils.isObject(entity[prop])) {
+        return helper(entity[prop]).toPOJO() as T[keyof T];
+      }
+    }
+
     const customType = property?.customType;
 
     if (customType) {

packages/knex/src/AbstractSqlDriver.ts

@@ -860,6 +860,25 @@ export abstract class AbstractSqlDriver<C extends AbstractSqlConnection = Abstra
       ret.push(...this.getFieldsForJoinedLoad(qb, meta, fields, populate));
     } else if (fields) {
       for (const field of [...fields]) {
+        if (field.toString().includes('.')) {
+          const parts = fields.toString().split('.');
+          const rootPropName = parts.shift()!; // first one is the `prop`
+          const prop = QueryHelper.findProperty(rootPropName, {
+            metadata: this.metadata,
+            platform: this.platform,
+            entityName: meta.className,
+            where: {},
+            aliasMap: qb.getAliasMap(),
+          });
+
+          if (prop?.reference === ReferenceType.EMBEDDED) {
+            const nest = (p: EntityProperty): EntityProperty => parts.length > 0 ? nest(p.embeddedProps[parts.shift()!]) : p;
+            const childProp = nest(prop);
+            ret.push(childProp.fieldNames[0]);
+            continue;
+          }
+        }
+
         if (Utils.isPlainObject(field) || field.toString().includes('.')) {
           continue;
         }

packages/knex/src/query/QueryBuilder.ts

@@ -1,9 +1,9 @@
 import type { Knex } from 'knex';
 import type {
-  AnyEntity, ConnectionType, Dictionary, EntityData, EntityMetadata, EntityProperty, FlatQueryOrderMap, RequiredEntityData, ObjectQuery,
-  GroupOperator, MetadataStorage, PopulateOptions, QBFilterQuery, QueryOrderMap, QueryResult, FlushMode, FilterQuery, QBQueryOrderMap,
+  AnyEntity, ConnectionType, Dictionary, EntityData, EntityMetadata, EntityProperty, FilterQuery, FlatQueryOrderMap, FlushMode, GroupOperator,
+  MetadataStorage, ObjectQuery, PopulateOptions, QBFilterQuery, QBQueryOrderMap, QueryOrderMap, QueryResult, RequiredEntityData,
 } from '@mikro-orm/core';
-import { LoadStrategy, LockMode, PopulateHint, QueryFlag, QueryHelper, ReferenceType, Utils, ValidationError } from '@mikro-orm/core';
+import { LoadStrategy, LockMode, PopulateHint, QueryFlag, QueryHelper, ReferenceType, Utils, ValidationError, helper } from '@mikro-orm/core';
 import { QueryType } from './enums';
 import type { AbstractSqlDriver } from '../AbstractSqlDriver';
 import { QueryBuilderHelper } from './QueryBuilderHelper';
@@ -280,9 +280,9 @@ export class QueryBuilder<T extends object = AnyEntity> {
     return this;
   }
 
-  onConflict(fields: string | string[] = []): this {
+  onConflict(fields: Field<T> | Field<T>[] = []): this {
     this._onConflict = this._onConflict || [];
-    this._onConflict.push({ fields: Utils.asArray(fields) });
+    this._onConflict.push({ fields: Utils.asArray(fields).map(f => f.toString()) });
     return this;
   }
 
@@ -707,6 +707,31 @@ export class QueryBuilder<T extends object = AnyEntity> {
         return;
       }
 
+      if (prop?.embedded) {
+        const fieldName = this.helper.mapper(prop.fieldNames[0], this.type) as string;
+        ret.push(fieldName);
+        return;
+      }
+
+      if (prop?.reference === ReferenceType.EMBEDDED) {
+        if (prop.object) {
+          ret.push(this.helper.mapper(prop.fieldNames[0], this.type) as string);
+        } else {
+          const nest = (prop: EntityProperty): void => {
+            for (const childProp of Object.values(prop.embeddedProps)) {
+              if (childProp.fieldNames) {
+                ret.push(this.helper.mapper(childProp.fieldNames[0], this.type) as string);
+              } else {
+                nest(childProp);
+              }
+            }
+          };
+          nest(prop);
+        }
+
+        return;
+      }
+
       ret.push(this.helper.mapper(field, this.type) as string);
     });
 
@@ -720,7 +745,8 @@ export class QueryBuilder<T extends object = AnyEntity> {
 
     Object.keys(this._populateMap).forEach(f => {
       if (!fields.includes(f.replace(/#\w+$/, '')) && type === 'where') {
-        ret.push(...this.helper.mapJoinColumns(this.type, this._joins[f]) as string[]);
+        const cols = this.helper.mapJoinColumns(this.type, this._joins[f]);
+        ret.push(...cols as string[]);
       }
 
       if (this._joins[f].prop.reference !== ReferenceType.ONE_TO_ONE && this._joins[f].inverseJoinColumns) {
@@ -746,6 +772,10 @@ export class QueryBuilder<T extends object = AnyEntity> {
     }
 
     if (data) {
+      if (Utils.isEntity(data)) {
+        data = helper(data).toJSON();
+      }
+
       this._data = this.helper.processData(data, this.flags.has(QueryFlag.CONVERT_CUSTOM_TYPES));
     }
 

packages/knex/src/query/QueryBuilderHelper.ts

@@ -52,6 +52,12 @@ export class QueryBuilderHelper {
     const customExpression = QueryBuilderHelper.isCustomExpression(field, !!alias);
     const [a, f] = this.splitField(field);
     const prop = this.getProperty(f, a);
+
+    // embeddable nested path instead of a regular property with table alias, reset alias
+    if (prop?.name === a && prop.embeddedProps[f]) {
+      return this.alias + '.' + prop.fieldNames[0];
+    }
+
     const noPrefix = prop && prop.persist === false;
 
     if (prop?.fieldNameRaw) {
@@ -522,7 +528,7 @@ export class QueryBuilderHelper {
       }
 
       // eslint-disable-next-line prefer-const
-      let [alias, field] = this.splitField(k);
+      let [alias, field] = this.splitField(k, true);
       alias = populate[alias] || alias;
 
       Utils.splitPrimaryKeys(field).forEach(f => {
@@ -553,10 +559,21 @@ export class QueryBuilderHelper {
     }
   }
 
-  splitField(field: string): [string, string] {
+  splitField(field: string, greedyAlias = false): [string, string] {
     const parts = field.split('.');
-    const fromField = parts.pop()!;
-    const fromAlias = parts.length > 0 ? parts.join('.') : this.alias;
+
+    if (parts.length === 1) {
+      return [this.alias, parts[0]];
+    }
+
+    if (greedyAlias) {
+      const fromField = parts.pop()!;
+      const fromAlias = parts.join('.');
+      return [fromAlias, fromField];
+    }
+
+    const fromAlias = parts.shift()!;
+    const fromField = parts.join('.');
 
     return [fromAlias, fromField];
   }
@@ -615,7 +632,8 @@ export class QueryBuilderHelper {
 
       ret = alias + fieldName;
     } else {
-      const [a, f] = field.split('.');
+      const [a, ...rest] = field.split('.');
+      const f = rest.join('.');
       ret = a + '.' + this.fieldName(f, a);
     }
 
@@ -673,14 +691,30 @@ export class QueryBuilderHelper {
     }
 
     /* istanbul ignore next */
-    return prop.fieldNames[0] ?? field;
+    return prop.fieldNames?.[0] ?? field;
   }
 
   getProperty(field: string, alias?: string): EntityProperty | undefined {
     const entityName = this.aliasMap[alias!] || this.entityName;
     const meta = this.metadata.find(entityName);
 
-    return meta ? meta.properties[field] : undefined;
+    // check if `alias` is not matching an embedded property name instead of alias, e.g. `address.city`
+    if (alias && meta) {
+      const prop = meta.properties[alias];
+
+      if (prop?.reference === ReferenceType.EMBEDDED) {
+        // we want to select the full object property so hydration works as expected
+        if (prop.object) {
+          return prop;
+        }
+
+        const parts = field.split('.');
+        const nest = (p: EntityProperty): EntityProperty => parts.length > 0 ? nest(p.embeddedProps[parts.shift()!]) : p;
+        return nest(prop);
+      }
+    }
+
+    return meta?.properties[field];
   }
 
   isTableNameAliasRequired(type: QueryType): boolean {

packages/knex/src/typings.ts

@@ -13,7 +13,10 @@ export type KnexStringRef = Knex.Ref<string, {
   [alias: string]: string;
 }>;
 
-export type Field<T> = string | keyof T | KnexStringRef | Knex.QueryBuilder;
+// eslint-disable-next-line @typescript-eslint/ban-types
+type AnyString = string & {};
+
+export type Field<T> = AnyString | keyof T | KnexStringRef | Knex.QueryBuilder;
 
 export interface JoinOptions {
   table: string;

tests/features/embeddables/__snapshots__/embedded-entities.postgres.test.ts.snap

@@ -1,7 +1,8 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`embedded entities in postgresql schema: embeddables 1 1`] = `
-"create table "user" ("id" serial primary key, "address1_street" varchar(255) not null, "address1_number" int not null, "address1_rank" real null, "address1_postal_code" varchar(255) not null, "address1_city" varchar(255) not null, "address1_country" varchar(255) not null, "addr_street" varchar(255) null, "addr_postal_code" varchar(255) null, "addr_city" varchar(255) null, "addr_country" varchar(255) null, "street" varchar(255) not null, "number" int not null, "rank" real null, "postal_code" varchar(255) not null, "city" varchar(255) not null, "country" varchar(255) not null, "address4" jsonb not null, "addresses" jsonb not null, "after" int null);
+"create table "user" ("id" serial primary key, "email" varchar(255) not null, "address1_street" varchar(255) not null, "address1_number" int not null, "address1_rank" real null, "address1_postal_code" varchar(255) not null, "address1_city" varchar(255) not null, "address1_country" varchar(255) not null, "addr_street" varchar(255) null, "addr_postal_code" varchar(255) null, "addr_city" varchar(255) null, "addr_country" varchar(255) null, "street" varchar(255) not null, "number" int not null, "rank" real null, "postal_code" varchar(255) not null, "city" varchar(255) not null, "country" varchar(255) not null, "address4" jsonb not null, "addresses" jsonb not null, "after" int null);
+alter table "user" add constraint "user_email_unique" unique ("email");
 
 "
 `;

tests/features/embeddables/embedded-entities.postgres.test.ts

@@ -63,6 +63,9 @@ class User {
   @PrimaryKey()
   id!: number;
 
+  @Property({ unique: true })
+  email!: string;
+
   @Embedded()
   address1!: Address1;
 
@@ -147,6 +150,7 @@ describe('embedded entities in postgresql', () => {
 
   test('assigning to array embeddables (GH #1699)', async () => {
     const user = new User();
+    user.email = 'test';
     expect(user.addresses).toEqual([]);
     const address1 = new Address1('Downing street 13A', 10, '10A', 'London 4A', 'UK 4A');
     const address2 = { street: 'Downing street 23A', number: 20, postalCode: '20A', city: 'London 24A', country: 'UK 24A' };
@@ -170,8 +174,9 @@ describe('embedded entities in postgresql', () => {
     expect(user.addresses).toHaveLength(2);
   });
 
-  test('persist and load', async () => {
-    const user = orm.em.create(User, {
+  function createUser() {
+    return orm.em.create(User, {
+      email: `test-${Math.random()}`,
       address1: { street: 'Downing street 10', number: 10, postalCode: '123', city: 'London 1', country: 'UK 1' },
       address2: { street: 'Downing street 11', city: 'London 2', country: 'UK 2' },
       address3: { street: 'Downing street 12', number: 10, postalCode: '789', city: 'London 3', country: 'UK 3' },
@@ -181,12 +186,15 @@ describe('embedded entities in postgresql', () => {
         { street: 'Downing street 13B', number: 10, postalCode: '10B', city: 'London 4B', country: 'UK 4B' },
       ],
     });
+  }
 
+  test('persist and load', async () => {
     const mock = mockLogger(orm, ['query']);
+    const user = createUser();
     await orm.em.persistAndFlush(user);
     orm.em.clear();
     expect(mock.mock.calls[0][0]).toMatch('begin');
-    expect(mock.mock.calls[1][0]).toMatch('insert into "user" ("addr_city", "addr_country", "addr_postal_code", "addr_street", "address1_city", "address1_country", "address1_number", "address1_postal_code", "address1_rank", "address1_street", "address4", "addresses", "city", "country", "number", "postal_code", "rank", "street") values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) returning "id"');
+    expect(mock.mock.calls[1][0]).toMatch('insert into "user" ("addr_city", "addr_country", "addr_postal_code", "addr_street", "address1_city", "address1_country", "address1_number", "address1_postal_code", "address1_rank", "address1_street", "address4", "addresses", "city", "country", "email", "number", "postal_code", "rank", "street") values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) returning "id"');
     expect(mock.mock.calls[2][0]).toMatch('commit');
 
     const u = await orm.em.findOneOrFail(User, user.id);
@@ -256,16 +264,59 @@ describe('embedded entities in postgresql', () => {
     const u5 = await orm.em.findOneOrFail(User, { address4: { number: { $gt: 2 } } });
     expect(u5).toBe(u1);
     expect(mock.mock.calls[11][0]).toMatch('select "u0".* from "user" as "u0" where ("u0"."address4"->>\'number\')::float8 > $1 limit $2');
+  });
+
+  test('partial loading', async () => {
+    const mock = mockLogger(orm, ['query']);
+
+    await orm.em.fork().qb(User).select('address1.city').where({ address1: { city: 'London 1' } }).execute();
+    expect(mock.mock.calls[0][0]).toMatch('select "u0"."address1_city" from "user" as "u0" where "u0"."address1_city" = $1');
+
+    await orm.em.fork().findOne(User, { address1: { city: 'London 1' } }, { fields: ['address1.city'] });
+    expect(mock.mock.calls[1][0]).toMatch('select "u0"."id", "u0"."address1_city" from "user" as "u0" where "u0"."address1_city" = $1 limit $2');
+
+    await orm.em.fork().qb(User).select('address1').where({ address1: { city: 'London 1' } }).execute();
+    expect(mock.mock.calls[2][0]).toMatch('select "u0"."address1_street", "u0"."address1_number", "u0"."address1_rank", "u0"."address1_postal_code", "u0"."address1_city", "u0"."address1_country" from "user" as "u0" where "u0"."address1_city" = $1');
+
+    await orm.em.fork().findOne(User, { address1: { city: 'London 1' } }, { fields: ['address1'] });
+    expect(mock.mock.calls[3][0]).toMatch('select "u0"."id", "u0"."address1_street", "u0"."address1_number", "u0"."address1_rank", "u0"."address1_postal_code", "u0"."address1_city", "u0"."address1_country" from "user" as "u0" where "u0"."address1_city" = $1 limit $2');
+
+    mock.mockReset();
+
+    await orm.em.fork().qb(User).select('address4.city').where({ address4: { city: 'London 1' } }).execute(); // object embedded prop does not support nested partial loading
+    expect(mock.mock.calls[0][0]).toMatch(`select "u0"."address4" from "user" as "u0" where "u0"."address4"->>'city' = $1`);
+
+    await orm.em.fork().findOne(User, { address4: { city: 'London 1' } }, { fields: ['address4.city'] }); // object embedded prop does not support nested partial loading
+    expect(mock.mock.calls[1][0]).toMatch(`select "u0"."id", "u0"."address4" from "user" as "u0" where "u0"."address4"->>'city' = $1 limit $2`);
+
+    await orm.em.fork().qb(User).select('address4').where({ address4: { city: 'London 1' } }).execute();
+    expect(mock.mock.calls[2][0]).toMatch(`select "u0"."address4" from "user" as "u0" where "u0"."address4"->>'city' = $1`);
+
+    await orm.em.fork().findOne(User, { address4: { city: 'London 1' } }, { fields: ['address4'] });
+    expect(mock.mock.calls[3][0]).toMatch(`select "u0"."id", "u0"."address4" from "user" as "u0" where "u0"."address4"->>'city' = $1 limit $2`);
+
+    mock.mockReset();
+
+    await orm.em.fork().qb(User).select('addresses.city').where({ addresses: { city: 'London 1' } }).execute(); // object embedded prop does not support nested partial loading
+    expect(mock.mock.calls[0][0]).toMatch(`select "u0"."addresses" from "user" as "u0" where "u0"."addresses"->>'city' = $1`);
+
+    await orm.em.fork().findOne(User, { addresses: { city: 'London 1' } }, { fields: ['addresses.city'] }); // object embedded prop does not support nested partial loading
+    expect(mock.mock.calls[1][0]).toMatch(`select "u0"."id", "u0"."addresses" from "user" as "u0" where "u0"."addresses"->>'city' = $1 limit $2`);
+
+    await orm.em.fork().qb(User).select('addresses').where({ addresses: { city: 'London 1' } }).execute();
+    expect(mock.mock.calls[2][0]).toMatch(`select "u0"."addresses" from "user" as "u0" where "u0"."addresses"->>'city' = $1`);
+
+    await orm.em.fork().findOne(User, { addresses: { city: 'London 1' } }, { fields: ['addresses'] });
+    expect(mock.mock.calls[3][0]).toMatch(`select "u0"."id", "u0"."addresses" from "user" as "u0" where "u0"."addresses"->>'city' = $1 limit $2`);
 
-    // await orm.em.fork().findOneOrFail(User, { address1: { city: 'London 1' } }, { fields: ['address1.city'] });
-    // expect(mock.mock.calls[12][0]).toMatch('select "u0"."id", "u0"."address1_city" from "user" as "u0" where "u0"."address1_city" = $1 limit $2');
-    //
-    // await orm.em.fork().findOneOrFail(User, { address1: { city: 'London 1' } }, { fields: ['address1'] });
-    // expect(mock.mock.calls[13][0]).toMatch('select "u0".* from "user" as "u0" where ("u0"."address4"->>\'number\')::float8 > $1 limit $2');
+    const user = createUser();
+    await orm.em.fork().qb(User).insert(user).onConflict(['email']).merge(['email', 'address1.city']);
+    expect(mock.mock.calls[4][0]).toMatch(`insert into "user" ("addr_city", "addr_country", "addr_street", "address1_city", "address1_country", "address1_number", "address1_postal_code", "address1_street", "address4", "addresses", "city", "country", "email", "number", "postal_code", "street") values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) on conflict ("email") do update set "email" = excluded."email", "address1_city" = excluded."address1_city" returning "id"`);
   });
 
   test('assign', async () => {
     const user = new User();
+    user.email = `test-${Math.random()}`;
     orm.em.assign(user, {
       address1: { street: 'Downing street 10', postalCode: '123', city: 'London 1', country: 'UK 1' },
       address2: { street: 'Downing street 11', city: 'London 2', country: 'UK 2' },
@@ -299,6 +350,7 @@ describe('embedded entities in postgresql', () => {
   test('native update entity', async () => {
     const user = new User();
     orm.em.assign(user, {
+      email: 'test',
       address1: { street: 'Downing street 10', number: 3, postalCode: '123', city: 'London 1', country: 'UK 1' },
       address2: { street: 'Downing street 11', city: 'London 2', country: 'UK 2' },
       address3: { street: 'Downing street 12', number: 3, postalCode: '789', city: 'London 3', country: 'UK 3' },
@@ -324,6 +376,7 @@ describe('embedded entities in postgresql', () => {
 
   test('query by complex custom expressions with JSON operator and casting (GH issue 1261)', async () => {
     const user = new User();
+    user.email = `test-${Math.random()}`;
     user.address1 = new Address1('Test', 10, '12000', 'Prague', 'CZ');
     user.address3 = new Address1('Test', 10, '12000', 'Prague', 'CZ');
     user.address4 = new Address1('Test', 10, '12000', 'Prague', 'CZ');