feat(core): add sql.ref() helper (#4402)

Replacement for `qb.ref()`.

docs/docs/entity-manager.md

@@ -369,13 +369,11 @@ try {
 
 ### Using custom SQL fragments
 
-It is possible to use any SQL fragment in our `WHERE` query or `ORDER BY` clause:
-
-> The `expr()` helper is an identity function - all it does is to return its parameter. We can use it to bypass the strict type checks in `FilterQuery`.
+Any SQL fragment in your `WHERE` query or `ORDER BY` clause need to be wrapped with `raw()` or `sql`:
 
 ```ts
-const users = await em.find(User, { [expr('lower(email)')]: 'foo@bar.baz' }, {
-  orderBy: { [`(point(loc_latitude, loc_longitude) <@> point(0, 0))`]: 'ASC' },
+const users = await em.find(User, { [sql`lower(email)`]: 'foo@bar.baz' }, {
+  orderBy: { [sql`(point(loc_latitude, loc_longitude) <@> point(0, 0))`]: 'ASC' },
 });
 ```
 
@@ -388,6 +386,8 @@ where lower(email) = 'foo@bar.baz'
 order by (point(loc_latitude, loc_longitude) <@> point(0, 0)) asc
 ```
 
+Read more about this in [Using raw SQL query fragments](./raw-queries.md) section.
+
 ## Updating references (not loaded entities)
 
 Since v5.5, we can update references via Unit of Work, just like if it was a loaded entity. This way it is possible to issue update queries without loading the entity.

docs/docs/query-builder.md

@@ -198,13 +198,13 @@ There are multiple ways to construct complex query conditions. You can either wr
 
 ### Using custom SQL fragments
 
-It is possible to use any SQL fragment in your `WHERE` query or `ORDER BY` clause:
+Any SQL fragment in your `WHERE` query or `ORDER BY` clause need to be wrapped with `raw()` or `sql`:
 
 ```ts
 const users = em.createQueryBuilder(User)
   .select('*')
-  .where({ 'lower(email)': 'foo@bar.baz' })
-  .orderBy({ [`(point(loc_latitude, loc_longitude) <@> point(0, 0))`]: 'ASC' })
+  .where({ [sql`lower(email)`]: 'foo@bar.baz' }) // sql tagged template function
+  .orderBy({ [raw(`(point(loc_latitude, loc_longitude) <@> point(0, 0))`)]: 'ASC' }) // raw helper
   .getResultList();
 ```
 
@@ -217,6 +217,8 @@ where lower(email) = 'foo@bar.baz'
 order by (point(loc_latitude, loc_longitude) <@> point(0, 0)) asc
 ```
 
+Read more about this in [Using raw SQL query fragments](./raw-queries.md) section.
+
 ### Custom SQL in where
 
 ```ts

docs/docs/raw-queries.md

@@ -45,5 +45,17 @@ await em.find(User, { time: sql`now()` });
 await em.find(User, { [sql`lower(name)`]: name.toLowerCase() });
 
 // value can be empty array
-await em.find(User, { [sql`(select 1 = 1)`]: [] });
+await em.find(User, { [sql`(select ${1} = ${1})`]: [] });
 ```
+
+### `sql.ref()`
+
+When you want to refer to a column, you can use the `sql.ref()` function:
+
+```ts
+await em.find(User, { foo: sql`bar` });
+```
+
+### Aliasing
+
+To select a raw fragment, we need to alias it. For that, we can use ```sql`(select 1 + 1)`.as('<alias>')```.

docs/docs/upgrading-v5-to-v6.md

@@ -304,6 +304,20 @@ await em.flush();
 console.log(ref.age); // real value is available after flush
 ```
 
+Alternatively, you can use the new `sql` tagged template function:
+
+```ts
+ref.age = sql`age * 2`;
+```
+
+This works on query keys as well as parameters, and is required for any SQL fragments.
+
+Read more about this in [Using raw SQL query fragments](./raw-queries.md) section.
+
+## Removed `qb.ref()`
+
+Removed in favour of `sql.ref()`.
+
 ## Changed default PostgreSQL `Date` mapping precision
 
 Previously, all drivers defaulted the `Date` type mapping to a timestamp with 0 precision (so seconds). This is [discouraged in PostgreSQL](https://wiki.postgresql.org/wiki/Don't_Do_This#Don.27t_use_timestamp.280.29_or_timestamptz.280.29), and is no longer valid - the default mapping without the `length` property being explicitly set is now `timestamptz`, which stores microsecond precision, so equivalent to `timestampz(6)`.

packages/core/src/utils/RawQueryFragment.ts

@@ -17,6 +17,10 @@ export class RawQueryFragment {
     this.#key = `[raw]: ${this.sql}${this.params ? ` (#${RawQueryFragment.#index++})` : ''}`;
   }
 
+  as(alias: string): RawQueryFragment {
+    return new RawQueryFragment(`${this.sql} as ${alias}`, this.params);
+  }
+
   valueOf(): string {
     throw new Error(`Trying to modify raw SQL fragment: '${this.sql}'`);
   }
@@ -124,6 +128,10 @@ export function raw<T extends object = any, R = any>(sql: EntityKey<T> | EntityK
     sql = sql(ALIAS_REPLACEMENT);
   }
 
+  if (sql === '??' && Array.isArray(params)) {
+    return new RawQueryFragment(sql, params) as R;
+  }
+
   if (Array.isArray(sql)) {
     // for composite FK we return just a simple string
     return Utils.getPrimaryKeyHash(sql) as R;
@@ -164,3 +172,5 @@ export function sql(sql: readonly string[], ...values: unknown[]) {
     return valueExists ? text + '?' : text;
   }, ''), values);
 }
+
+sql.ref = <T extends object>(...keys: string[]) => raw<T, RawQueryFragment>('??', [keys.join('.')]);

packages/knex/src/AbstractSqlDriver.ts

@@ -41,6 +41,7 @@ import {
   type QueryOrderMap,
   type QueryResult,
   raw,
+  sql,
   ReferenceKind,
   type RequiredEntityData,
   type Transaction,
@@ -885,20 +886,21 @@ export abstract class AbstractSqlDriver<Connection extends AbstractSqlConnection
    * @internal
    */
   mapPropToFieldNames<T extends object>(qb: QueryBuilder<T>, prop: EntityProperty<T>, tableAlias?: string): Field<T>[] {
-    const aliased = qb.ref(tableAlias ? `${tableAlias}__${prop.fieldNames[0]}` : prop.fieldNames[0]).toString();
+    const knex = this.connection.getKnex();
+    const aliased = knex.ref(tableAlias ? `${tableAlias}__${prop.fieldNames[0]}` : prop.fieldNames[0]).toString();
 
     if (prop.customType?.convertToJSValueSQL && tableAlias) {
-      const prefixed = qb.ref(prop.fieldNames[0]).withSchema(tableAlias).toString();
+      const prefixed = knex.ref(prop.fieldNames[0]).withSchema(tableAlias).toString();
       return [raw(`${prop.customType.convertToJSValueSQL(prefixed, this.platform)} as ${aliased}`)];
     }
 
     if (prop.formula) {
-      const alias = qb.ref(tableAlias ?? qb.alias).toString();
+      const alias = knex.ref(tableAlias ?? qb.alias).toString();
       return [raw(`${prop.formula!(alias)} as ${aliased}`)];
     }
 
     if (tableAlias) {
-      return prop.fieldNames.map(fieldName => qb.ref(fieldName).withSchema(tableAlias).as(`${tableAlias}__${fieldName}`));
+      return prop.fieldNames.map(fieldName => knex.ref(fieldName).withSchema(tableAlias).as(`${tableAlias}__${fieldName}`));
     }
 
     return prop.fieldNames;
@@ -1146,8 +1148,8 @@ export abstract class AbstractSqlDriver<Connection extends AbstractSqlConnection
       meta.props
         .filter(prop => prop.formula && !lazyProps.includes(prop))
         .forEach(prop => {
-          const alias = qb.ref(qb.alias).toString();
-          const aliased = qb.ref(prop.fieldNames[0]).toString();
+          const alias = this.connection.getKnex().ref(qb.alias).toString();
+          const aliased = this.connection.getKnex().ref(prop.fieldNames[0]).toString();
           ret.push(raw(`${prop.formula!(alias)} as ${aliased}`));
         });
 

packages/knex/src/AbstractSqlPlatform.ts

@@ -61,9 +61,14 @@ export abstract class AbstractSqlPlatform extends Platform {
     let pos = 0;
     let ret = '';
 
-    if (sql[0] === '?' && sql[1] !== '?') {
-      ret += this.quoteValue(params[j++]);
-      pos = 1;
+    if (sql[0] === '?') {
+      if (sql[1] === '?') {
+        ret += this.quoteIdentifier(params[j++]);
+        pos = 2;
+      } else {
+        ret += this.quoteValue(params[j++]);
+        pos = 1;
+      }
     }
 
     while (pos < sql.length) {

packages/knex/src/query/QueryBuilder.ts

@@ -411,13 +411,6 @@ export class QueryBuilder<T extends object = AnyEntity> {
     return this;
   }
 
-  /**
-   * @internal
-   */
-  ref(field: string) {
-    return this.knex.ref(field);
-  }
-
   limit(limit?: number, offset = 0): this {
     this.ensureNotFinalized();
     this._limit = limit;
@@ -1240,7 +1233,7 @@ export class QueryBuilder<T extends object = AnyEntity> {
             addToSelect.push(fieldName);
           }
 
-          orderBy.push({ [raw(`min(${this.ref(fieldName)}${type})`)]: direction });
+          orderBy.push({ [raw(`min(${this.knex.ref(fieldName)}${type})`)]: direction });
         }
       }
 

packages/knex/src/query/QueryBuilderHelper.ts

@@ -456,6 +456,10 @@ export class QueryBuilderHelper {
   private appendQuerySubCondition(qb: Knex.QueryBuilder, type: QueryType, method: 'where' | 'having', cond: any, key: string, operator?: '$and' | '$or'): void {
     const m = operator === '$or' ? 'orWhere' : method;
 
+    if (cond[key] instanceof RawQueryFragment) {
+      cond[key] = this.knex.raw(cond[key].sql, cond[key].params);
+    }
+
     if (this.isSimpleRegExp(cond[key])) {
       return void qb[m](this.mapper(key, type), 'like', this.getRegExpParam(cond[key]));
     }
@@ -589,10 +593,14 @@ export class QueryBuilderHelper {
         const rawColumn = Utils.isString(column) ? column.split('.').map(e => this.knex.ref(e)).join('.') : column;
         const customOrder = prop?.customOrder;
 
-        const colPart = customOrder
+        let colPart = customOrder
           ? this.platform.generateCustomOrder(rawColumn, customOrder)
           : rawColumn;
 
+        if (Utils.isRawSql(colPart)) {
+          colPart = this.platform.formatQuery(colPart.sql, colPart.params);
+        }
+
         ret.push(`${colPart} ${order.toLowerCase()}`);
       });
     });

packages/knex/src/typings.ts

@@ -1,5 +1,5 @@
 import type { Knex } from 'knex';
-import type { CheckCallback, Dictionary, EntityProperty, GroupOperator, QBFilterQuery, QueryOrderMap, Type } from '@mikro-orm/core';
+import type { CheckCallback, Dictionary, EntityProperty, GroupOperator, RawQueryFragment, QBFilterQuery, QueryOrderMap, Type } from '@mikro-orm/core';
 import type { QueryType } from './query/enums';
 import type { DatabaseSchema, DatabaseTable } from './schema';
 
@@ -15,7 +15,7 @@ export type KnexStringRef = Knex.Ref<string, {
 
 type AnyString = string & {};
 
-export type Field<T> = AnyString | keyof T | KnexStringRef | Knex.QueryBuilder;
+export type Field<T> = AnyString | keyof T | RawQueryFragment | KnexStringRef | Knex.QueryBuilder;
 
 export interface JoinOptions {
   table: string;

tests/QueryBuilder.test.ts

@@ -682,7 +682,7 @@ describe('QueryBuilder', () => {
 
   test('GH #4104', async () => {
     const qb = orm.em.createQueryBuilder(Author2, 'a');
-    const qb1 = orm.em.createQueryBuilder(Book2, 'b').count('b.uuid', true).where({ author: qb.ref('a.id') }).as('Author2.booksTotal');
+    const qb1 = orm.em.createQueryBuilder(Book2, 'b').count('b.uuid', true).where({ author: sql.ref('a.id') }).as('Author2.booksTotal');
     qb.select(['*', qb1])
       .where({ books: { title: 'foo' } })
       .limit(1)
@@ -901,18 +901,20 @@ describe('QueryBuilder', () => {
 
   test('select with group by and having', async () => {
     const qb = orm.em.createQueryBuilder(BookTag2, 't');
-    qb.select(['b.*', 't.*', raw('count(t.id) as tags')])
+    qb.select(['b.*', 't.*', sql`count(t.id)`.as('tags')])
+      .addSelect(sql.ref('b.title').as('book_title'))
       .leftJoin('t.books', 'b')
       .where('b.title = ? or b.title = ?', ['test 123', 'lol 321'])
       .groupBy(['b.uuid', 't.id'])
       .having('tags > ?', [0]);
-    const sql = 'select `b`.*, `t`.*, count(t.id) as tags from `book_tag2` as `t` ' +
+    const query = 'select `b`.*, `t`.*, count(t.id) as tags, `b`.`title` as book_title ' +
+      'from `book_tag2` as `t` ' +
       'left join `book2_tags` as `e1` on `t`.`id` = `e1`.`book_tag2_id` ' +
       'left join `book2` as `b` on `e1`.`book2_uuid_pk` = `b`.`uuid_pk` ' +
       'where (b.title = ? or b.title = ?) ' +
       'group by `b`.`uuid_pk`, `t`.`id` ' +
       'having (tags > ?)';
-    expect(qb.getQuery()).toEqual(sql);
+    expect(qb.getQuery()).toEqual(query);
     expect(qb.getParams()).toEqual(['test 123', 'lol 321', 0]);
   });
 
@@ -2033,13 +2035,13 @@ describe('QueryBuilder', () => {
 
   test('select with sub-query', async () => {
     const knex = orm.em.getKnex();
-    const qb1 = orm.em.createQueryBuilder(Book2, 'b').count('b.uuid', true).where({ author: knex.ref('a.id') }).as('Author2.booksTotal');
+    const qb1 = orm.em.createQueryBuilder(Book2, 'b').count('b.uuid', true).where({ author: sql.ref('a.id') }).as('Author2.booksTotal');
     const qb2 = orm.em.createQueryBuilder(Author2, 'a');
     qb2.select(['*', qb1]).orderBy({ booksTotal: 'desc' });
     expect(qb2.getQuery()).toEqual('select `a`.*, (select count(distinct `b`.`uuid_pk`) as `count` from `book2` as `b` where `b`.`author_id` = `a`.`id`) as `books_total` from `author2` as `a` order by `books_total` desc');
     expect(qb2.getParams()).toEqual([]);
 
-    const qb3 = orm.em.createQueryBuilder(Book2, 'b').count('b.uuid', true).where({ author: knex.ref('a.id') }).as('books_total');
+    const qb3 = orm.em.createQueryBuilder(Book2, 'b').count('b.uuid', true).where({ author: sql.ref('a.id') }).as('books_total');
     const qb4 = orm.em.createQueryBuilder(Author2, 'a');
     qb4.select(['*', qb3]).orderBy({ booksTotal: 'desc' });
     expect(qb4.getQuery()).toEqual('select `a`.*, (select count(distinct `b`.`uuid_pk`) as `count` from `book2` as `b` where `b`.`author_id` = `a`.`id`) as `books_total` from `author2` as `a` order by `books_total` desc');
@@ -2048,13 +2050,13 @@ describe('QueryBuilder', () => {
 
   test('select where sub-query', async () => {
     const knex = orm.em.getKnex();
-    const qb1 = orm.em.createQueryBuilder(Book2, 'b').count('b.uuid', true).where({ author: knex.ref('a.id') }).getKnexQuery();
+    const qb1 = orm.em.createQueryBuilder(Book2, 'b').count('b.uuid', true).where({ author: sql.ref('a.id') }).getKnexQuery();
     const qb2 = orm.em.createQueryBuilder(Author2, 'a');
     qb2.select('*').withSubQuery(qb1, 'a.booksTotal').where({ 'a.booksTotal': { $in: [1, 2, 3] } });
     expect(qb2.getQuery()).toEqual('select `a`.* from `author2` as `a` where (select count(distinct `b`.`uuid_pk`) as `count` from `book2` as `b` where `b`.`author_id` = `a`.`id`) in (?, ?, ?)');
     expect(qb2.getParams()).toEqual([1, 2, 3]);
 
-    const qb3 = orm.em.createQueryBuilder(Book2, 'b').count('b.uuid', true).where({ author: knex.ref('a.id') }).getKnexQuery();
+    const qb3 = orm.em.createQueryBuilder(Book2, 'b').count('b.uuid', true).where({ author: sql.ref('a.id') }).getKnexQuery();
     const qb4 = orm.em.createQueryBuilder(Author2, 'a');
     qb4.select('*').withSubQuery(qb3, 'a.booksTotal').where({ 'a.booksTotal': 1 });
     expect(qb4.getQuery()).toEqual('select `a`.* from `author2` as `a` where (select count(distinct `b`.`uuid_pk`) as `count` from `book2` as `b` where `b`.`author_id` = `a`.`id`) = ?');

tests/features/filters/filters.postgres.test.ts

@@ -10,7 +10,7 @@ import {
   PrimaryKey,
   Property,
   Ref,
-  raw,
+  sql,
 } from '@mikro-orm/core';
 import type { AbstractSqlDriver } from '@mikro-orm/knex';
 import { mockLogger } from '../../helpers';
@@ -100,7 +100,7 @@ class User {
 @Entity()
 @Filter({
   name: 'user',
-  cond: () => ({ user: { $or: [{ firstName: 'name' }, { lastName: 'name' }, { age: raw('(select 1 + 1)') }] } }),
+  cond: () => ({ user: { $or: [{ firstName: 'name' }, { lastName: 'name' }, { age: sql`(select ${1} + ${1})` }] } }),
   default: true,
   args: false,
 })
@@ -206,7 +206,7 @@ describe('filters [postgres]', () => {
     }, { filters: false });
 
     expect(mock.mock.calls[0][0]).toMatch(`select "u0".* from "user" as "u0" where ("u0"."age" = $1 or "u0"."age" = $2) and ("u0"."first_name" = $3 or "u0"."last_name" = $4)`);
-    expect(mock.mock.calls[1][0]).toMatch(`select "m0".* from "membership" as "m0" left join "user" as "u1" on "m0"."user_id" = "u1"."id" where ("u1"."first_name" = $1 or "u1"."last_name" = $2 or "u1"."age" = $3) and ("m0"."role" = $4 or "m0"."role" = $5)`);
+    expect(mock.mock.calls[1][0]).toMatch(`select "m0".* from "membership" as "m0" left join "user" as "u1" on "m0"."user_id" = "u1"."id" where ("u1"."first_name" = $1 or "u1"."last_name" = $2 or "u1"."age" = (select $3 + $4)) and ("m0"."role" = $5 or "m0"."role" = $6)`);
     expect(mock.mock.calls[2][0]).toMatch(`select "m0".* from "membership" as "m0" left join "user" as "u1" on "m0"."user_id" = "u1"."id" where ("m0"."role" = $1 or "m0"."role" = $2) and ("u1"."first_name" = $3 or "u1"."last_name" = $4)`);
   });
 

tests/features/schema-generator/native-enums.postgres.test.ts

@@ -64,6 +64,25 @@ describe('native enums in postgres', () => {
 
   afterAll(() => orm.close());
 
+  test('generate schema from metadata [postgres]', async () => {
+    orm.getMetadata().reset('NewTable');
+    orm.em.getConnection().execute('drop schema if exists different_schema cascade');
+    const dump = await orm.schema.getCreateSchemaSQL();
+    expect(dump).toMatchSnapshot('postgres-schema-dump');
+
+    const dropDump = await orm.schema.getDropSchemaSQL();
+    expect(dropDump).toMatchSnapshot('postgres-drop-schema-dump');
+    await orm.schema.execute(dropDump, { wrap: true });
+
+    const createDump = await orm.schema.getCreateSchemaSQL();
+    expect(createDump).toMatchSnapshot('postgres-create-schema-dump');
+    await orm.schema.execute(createDump, { wrap: true });
+
+    const updateDump = await orm.schema.getUpdateSchemaSQL();
+    expect(updateDump).toMatchSnapshot('postgres-update-schema-dump');
+    await orm.schema.execute(updateDump, { wrap: true });
+  });
+
   test('enum diffing', async () => {
     orm.em.getConnection().execute('drop schema if exists different_schema cascade');
     const newTableMeta = new EntitySchema({
@@ -123,23 +142,4 @@ describe('native enums in postgres', () => {
     await orm.schema.execute(diff);
   });
 
-  test('generate schema from metadata [postgres]', async () => {
-    orm.getMetadata().reset('NewTable');
-    orm.em.getConnection().execute('drop schema if exists different_schema cascade');
-    const dump = await orm.schema.getCreateSchemaSQL();
-    expect(dump).toMatchSnapshot('postgres-schema-dump');
-
-    const dropDump = await orm.schema.getDropSchemaSQL();
-    expect(dropDump).toMatchSnapshot('postgres-drop-schema-dump');
-    await orm.schema.execute(dropDump, { wrap: true });
-
-    const createDump = await orm.schema.getCreateSchemaSQL();
-    expect(createDump).toMatchSnapshot('postgres-create-schema-dump');
-    await orm.schema.execute(createDump, { wrap: true });
-
-    const updateDump = await orm.schema.getUpdateSchemaSQL();
-    expect(updateDump).toMatchSnapshot('postgres-update-schema-dump');
-    await orm.schema.execute(updateDump, { wrap: true });
-  });
-
 });