fix(core): support raw fragments in order by with pagination

Closes #5110

packages/core/src/utils/Cursor.ts

@@ -4,6 +4,7 @@ import type { FindByCursorOptions, OrderDefinition } from '../drivers/IDatabaseD
 import { Utils } from './Utils';
 import { ReferenceKind, type QueryOrder, type QueryOrderKeys } from '../enums';
 import { Reference } from '../entity/Reference';
+import { RawQueryFragment } from '../utils/RawQueryFragment';
 
 /**
  * As an alternative to the offset-based pagination with `limit` and `offset`, we can paginate based on a cursor.
@@ -155,10 +156,24 @@ export class Cursor<
 
   static getDefinition<Entity extends object>(meta: EntityMetadata<Entity>, orderBy: OrderDefinition<Entity>) {
     return Utils.asArray(orderBy).flatMap(order => {
-      return Utils.keys(order)
-        .map(key => meta.properties[key as EntityKey<Entity>])
-        .filter(prop => prop && ([ReferenceKind.SCALAR, ReferenceKind.EMBEDDED, ReferenceKind.MANY_TO_ONE].includes(prop.kind) || (prop.kind === ReferenceKind.ONE_TO_ONE && prop.owner)))
-        .map(prop => [prop.name, order[prop.name] as QueryOrder] as const);
+      const ret: [EntityKey, QueryOrder][] = [];
+
+      for (const key of Utils.keys(order)) {
+        if (RawQueryFragment.isKnownFragment(key)) {
+          ret.push([key as EntityKey, order[key] as QueryOrder]);
+          continue;
+        }
+
+        const prop = meta.properties[key];
+
+        if (!prop || !([ReferenceKind.SCALAR, ReferenceKind.EMBEDDED, ReferenceKind.MANY_TO_ONE].includes(prop.kind) || (prop.kind === ReferenceKind.ONE_TO_ONE && prop.owner))) {
+          continue;
+        }
+
+        ret.push([prop.name as EntityKey, order[prop.name] as QueryOrder]);
+      }
+
+      return ret;
     });
   }
 

packages/core/src/utils/RawQueryFragment.ts

@@ -47,6 +47,13 @@ export class RawQueryFragment {
     return new RawQueryFragment(this.sql, this.params);
   }
 
+  /**
+   * @internal allows testing we don't leak memory, as the raw fragments cache needs to be cleared automatically
+   */
+  static checkCacheSize() {
+    return this.#rawQueryCache.size;
+  }
+
   static isKnownFragment(key: string) {
     return this.#rawQueryCache.has(key);
   }

packages/knex/src/query/QueryBuilder.ts

@@ -913,22 +913,30 @@ export class QueryBuilder<T extends object = AnyEntity> {
     return ret;
   }
 
-  clone(reset?: boolean): QueryBuilder<T> {
+  clone(reset?: boolean | string[]): QueryBuilder<T> {
     const qb = new QueryBuilder<T>(this.mainAlias.entityName, this.metadata, this.driver, this.context, this.mainAlias.aliasName, this.connectionType, this.em);
 
-    if (reset) {
+    if (reset === true) {
       return qb;
     }
 
     Object.assign(qb, this);
+    reset = reset || [];
 
     // clone array/object properties
     const properties = [
       'flags', '_populate', '_populateWhere', '__populateWhere', '_populateMap', '_joins', '_joinedProps', '_cond', '_data', '_orderBy',
       '_schema', '_indexHint', '_cache', 'subQueries', 'lockMode', 'lockTables', '_groupBy', '_having', '_returning',
       '_comments', '_hintComments',
     ] as const;
-    properties.forEach(prop => (qb as any)[prop] = Utils.copy(this[prop]));
+
+    for (const prop of properties) {
+      if (reset.includes(prop)) {
+        continue;
+      }
+
+      (qb as any)[prop] = Utils.copy(this[prop]);
+    }
 
     /* istanbul ignore else */
     if (this._fields) {
@@ -1403,7 +1411,7 @@ export class QueryBuilder<T extends object = AnyEntity> {
 
   private wrapPaginateSubQuery(meta: EntityMetadata): void {
     const pks = this.prepareFields(meta.primaryKeys, 'sub-query') as string[];
-    const subQuery = this.clone().select(pks).groupBy(pks).limit(this._limit!);
+    const subQuery = this.clone(['_orderBy', '_cond']).select(pks).groupBy(pks).limit(this._limit!);
     // revert the on conditions added via populateWhere, we want to apply those only once
     Object.values(subQuery._joins).forEach(join => join.cond = join.cond_ ?? {});
 
@@ -1418,6 +1426,12 @@ export class QueryBuilder<T extends object = AnyEntity> {
 
       for (const orderMap of this._orderBy) {
         for (const [field, direction] of Object.entries(orderMap)) {
+          if (RawQueryFragment.isKnownFragment(field)) {
+            const rawField = RawQueryFragment.getKnownFragment(field, false)!;
+            orderBy.push({ [rawField.clone() as any]: direction });
+            continue;
+          }
+
           const [a, f] = this.helper.splitField(field as EntityKey<T>);
           const prop = this.helper.getProperty(f, a);
           const type = this.platform.castColumn(prop);
@@ -1427,7 +1441,8 @@ export class QueryBuilder<T extends object = AnyEntity> {
             addToSelect.push(fieldName);
           }
 
-          orderBy.push({ [raw(`min(${this.knex.ref(fieldName)}${type})`)]: direction });
+          const key = raw(`min(${this.knex.ref(fieldName)}${type})`);
+          orderBy.push({ [key]: direction });
         }
       }
 
@@ -1555,6 +1570,10 @@ export class QueryBuilder<T extends object = AnyEntity> {
       object.onConflict = this._onConflict[0];
     }
 
+    if (!Utils.isEmpty(this._orderBy)) {
+      object.orderBy = this._orderBy;
+    }
+
     const name = this._mainAlias ? `${prefix}QueryBuilder<${this._mainAlias?.entityName}>` : 'QueryBuilder';
     const ret = inspect(object, { depth });
 

tests/issues/GHx6.test.ts

@@ -1,4 +1,15 @@
-import { Entity, ManyToOne, MikroORM, PrimaryKey, Property, raw } from '@mikro-orm/sqlite';
+import {
+  Collection,
+  Entity,
+  ManyToMany,
+  ManyToOne,
+  MikroORM,
+  PrimaryKey,
+  Property,
+  QueryOrder,
+  raw,
+  RawQueryFragment,
+} from '@mikro-orm/sqlite';
 import { mockLogger } from '../helpers';
 
 @Entity()
@@ -27,6 +38,9 @@ class Tag {
   @ManyToOne(() => Job, { name: 'custom_name' })
   job!: Job;
 
+  @ManyToMany(() => Job)
+  jobs = new Collection<Job>(this);
+
 }
 
 let orm: MikroORM;
@@ -49,6 +63,7 @@ test('raw fragments with findAndCount', async () => {
     dateCompleted: { $ne: null },
     [raw(alias => `${alias}.DateCompleted`)]: '2023-07-24',
   });
+  expect(RawQueryFragment.checkCacheSize()).toBe(0);
 });
 
 test('raw fragments with orderBy', async () => {
@@ -59,6 +74,7 @@ test('raw fragments with orderBy', async () => {
     },
   });
   expect(mock.mock.calls[0][0]).toMatch('select `j0`.* from `job` as `j0` order by j0.DateCompleted desc');
+  expect(RawQueryFragment.checkCacheSize()).toBe(0);
 });
 
 test('raw fragments with orderBy on relation', async () => {
@@ -75,6 +91,7 @@ test('raw fragments with orderBy on relation', async () => {
     'from `tag` as `t0` ' +
     'left join `job` as `j1` on `t0`.`custom_name` = `j1`.`id` ' +
     'order by j1.DateCompleted desc');
+  expect(RawQueryFragment.checkCacheSize()).toBe(0);
 });
 
 test('raw fragments with populateOrderBy on relation', async () => {
@@ -92,6 +109,7 @@ test('raw fragments with populateOrderBy on relation', async () => {
     'from `tag` as `t0` ' +
     'left join `job` as `j1` on `t0`.`custom_name` = `j1`.`id` ' +
     'order by t0.created desc, j1.DateCompleted desc');
+  expect(RawQueryFragment.checkCacheSize()).toBe(0);
 });
 
 test('raw fragments with multiple items in filter', async () => {
@@ -102,4 +120,47 @@ test('raw fragments with multiple items in filter', async () => {
     },
   });
   expect(mock.mock.calls[0][0]).toMatch('select `t0`.* from `tag` as `t0` where id >= 10 and id <= 50');
+  expect(RawQueryFragment.checkCacheSize()).toBe(0);
+});
+
+test('qb.joinAndSelect', async () => {
+  const query = orm.em.qb(Tag, 'u')
+    .select('*')
+    .leftJoinAndSelect('jobs', 'a')
+    .where({
+      [raw('similarity("u"."name", ?)', ['abc'])]: { $gte: 0.3 },
+    })
+    .orderBy({
+      [raw('similarity(u."name", ?)', ['def'])]: QueryOrder.DESC_NULLS_LAST,
+    })
+    .limit(100)
+    .offset(0).
+    getFormattedQuery();
+  expect(query).toMatch('select `u`.*, `a`.`id` as `a__id`, `a`.`DateCompleted` as `a__DateCompleted` ' +
+    'from `tag` as `u` ' +
+    'left join `tag_jobs` as `t1` on `u`.`id` = `t1`.`tag_id` ' +
+    'left join `job` as `a` on `t1`.`job_id` = `a`.`id` ' +
+    'where `u`.`id` in (select `u`.`id` from (select `u`.`id` from `tag` as `u` left join `tag_jobs` as `t1` on `u`.`id` = `t1`.`tag_id` left join `job` as `a` on `t1`.`job_id` = `a`.`id` where similarity("u"."name", \'abc\') >= 0.3 group by `u`.`id` order by similarity(u."name", \'def\') desc nulls last limit 100) as `u`) ' +
+    'order by similarity(u."name", \'def\') desc nulls last');
+  expect(RawQueryFragment.checkCacheSize()).toBe(0);
+});
+
+test('em.findByCursor', async () => {
+  const mock = mockLogger(orm);
+  await orm.em.findByCursor(Tag, {}, {
+    populate: ['job'],
+    first: 3,
+    orderBy: {
+      [raw(alias => `${alias}.created`)]: 'desc',
+      job: {
+        [raw(alias => `${alias}.DateCompleted`)]: 'desc',
+      },
+    },
+  });
+  const queries = mock.mock.calls.flat().sort();
+  expect(queries[0]).toMatch('select `t0`.*, `j1`.`id` as `j1__id`, `j1`.`DateCompleted` as `j1__DateCompleted` ' +
+    'from `tag` as `t0` ' +
+    'left join `job` as `j1` on `t0`.`custom_name` = `j1`.`id` ' +
+    'order by t0.created desc, j1.DateCompleted desc');
+  expect(RawQueryFragment.checkCacheSize()).toBe(0);
 });