feat(sql): support $some, $none and $every subquery operators (#…

…4917)

In addition to the regular operators that translate to a real SQL
operator expression (e.g. `>=`), you can also use the following
collection operators:

| operator | description |

|----------|-----------------------------------------------------------------|
| `$some` | Finds collections that have some record matching the
condition. |
| `$none` | Finds collections that have no records matching the
condition. |
| `$every` | Finds collections where every record is matching the
condition. |

This will be resolved as a subquery condition:

```ts
// finds all authors that have some book called `Foo`
const res1 = await em.find(Author, {
  books: { $some: { title: 'Foo' } },
});

// finds all authors that have no books called `Foo`
const res2 = await em.find(Author, {
  books: { $none: { title: 'Foo' } },
});

// finds all authors that have every book called `Foo`
const res3 = await em.find(Author, {
  books: { $every: { title: 'Foo' } },
});
```

The condition object can be also empty:

```ts
// finds all authors that have at least one book
const res1 = await em.find(Author, {
  books: { $some: {} },
});

// finds all authors that have no books
const res2 = await em.find(Author, {
  books: { $none: {} },
});
```

Closes #2916

docs/docs/nested-populate.md

@@ -1,5 +1,5 @@
 ---
-title: Smart Nested Populate
+title: Nested Populate
 ---
 
 `MikroORM` is capable of loading large nested structures while maintaining good performance, querying each database table only once. Imagine you have this nested structure:

docs/docs/query-conditions.md

@@ -1,5 +1,5 @@
 ---
-title: Smart Query Conditions
+title: Query Conditions
 ---
 
 import Tabs from '@theme/Tabs';
@@ -77,6 +77,49 @@ const res = await orm.em.find(Author, [1, 2, 7]);
 | `$not`   | Inverts the effect of a query expression and returns documents that do not match the query expression.  |
 | `$or`    | Joins query clauses with a logical OR returns all documents that match the conditions of either clause. |
 
+### Collection
+
+In addition to the regular operators that translate to a real SQL operator expression (e.g. `>=`), you can also use the following collection operators:
+
+| operator | description                                                     |
+|----------|-----------------------------------------------------------------|
+| `$some`  | Finds collections that have some record matching the condition. |
+| `$none`  | Finds collections that have no records matching the condition.  |
+| `$every`  | Finds collections where every record is matching the condition. |
+
+This will be resolved as a subquery condition:
+
+```ts
+// finds all authors that have some book called `Foo`
+const res1 = await em.find(Author, {
+  books: { $some: { title: 'Foo' } },
+});
+
+// finds all authors that have no books called `Foo`
+const res2 = await em.find(Author, {
+  books: { $none: { title: 'Foo' } },
+});
+
+// finds all authors that have every book called `Foo`
+const res3 = await em.find(Author, {
+  books: { $every: { title: 'Foo' } },
+});
+```
+
+The condition object can be also empty:
+
+```ts
+// finds all authors that have at least one book
+const res1 = await em.find(Author, {
+  books: { $some: {} },
+});
+
+// finds all authors that have no books
+const res2 = await em.find(Author, {
+  books: { $none: {} },
+});
+```
+
 ## Regular Expressions
 
 The `$re` operator takes a string as input value, and by default uses the case-sensitive operator. If you would like to use a `RegExp` object, i.e. to be able to set flags, then search directly on the field name without using the operator:

docs/sidebars.js

@@ -34,6 +34,7 @@ module.exports = {
         'identity-map',
         'collections',
         'type-safe-relations',
+        'query-conditions',
         'repositories',
         'transactions',
         'inheritance-mapping',
@@ -56,7 +57,6 @@ module.exports = {
         'caching',
         'logging',
         'nested-populate',
-        'query-conditions',
         'propagation',
         'loading-strategies',
         'dataloaders',

packages/core/src/enums.ts

@@ -38,6 +38,9 @@ export enum QueryOperator {
   $overlap = '&&', // postgres only
   $contains = '@>', // postgres only
   $contained = '<@', // postgres only
+  $none = 'none', // collection operators, sql only
+  $some = 'some', // collection operators, sql only
+  $every = 'every', // collection operators, sql only
 }
 
 export const ARRAY_OPERATORS = [

packages/core/src/typings.ts

@@ -101,6 +101,9 @@ export type OperatorMap<T> = {
   $in?: ExpandScalar<T>[];
   $nin?: ExpandScalar<T>[];
   $not?: Query<T>;
+  $none?: Query<T>;
+  $some?: Query<T>;
+  $every?: Query<T>;
   $gt?: ExpandScalar<T>;
   $gte?: ExpandScalar<T>;
   $lt?: ExpandScalar<T>;

packages/knex/src/query/ArrayCriteriaNode.ts

@@ -12,6 +12,12 @@ export class ArrayCriteriaNode<T extends object> extends CriteriaNode<T> {
     });
   }
 
+  override unwrap(): any {
+    return this.payload.map((node: CriteriaNode<T>) => {
+      return node.unwrap();
+    });
+  }
+
   override willAutoJoin(qb: IQueryBuilder<T>, alias?: string) {
     return this.payload.some((node: CriteriaNode<T>) => {
       return node.willAutoJoin(qb, alias);

packages/knex/src/query/CriteriaNode.ts

@@ -43,6 +43,10 @@ export class CriteriaNode<T extends object> implements ICriteriaNode<T> {
     return this.payload;
   }
 
+  unwrap(): any {
+    return this.payload;
+  }
+
   shouldInline(payload: any): boolean {
     return false;
   }
@@ -56,7 +60,9 @@ export class CriteriaNode<T extends object> implements ICriteriaNode<T> {
     const composite = this.prop?.joinColumns ? this.prop.joinColumns.length > 1 : false;
     const customExpression = CriteriaNode.isCustomExpression(this.key!);
     const scalar = payload === null || Utils.isPrimaryKey(payload) || payload as unknown instanceof RegExp || payload as unknown instanceof Date || customExpression;
-    const operator = Utils.isPlainObject(payload) && Object.keys(payload).every(k => Utils.isOperator(k, false));
+    const plainObject = Utils.isPlainObject(payload);
+    const keys = plainObject ? Object.keys(payload) : [];
+    const operator = plainObject && keys.every(k => Utils.isOperator(k, false));
 
     if (composite) {
       return true;

packages/knex/src/query/ObjectCriteriaNode.ts

@@ -19,16 +19,42 @@ export class ObjectCriteriaNode<T extends object> extends CriteriaNode<T> {
   override process(qb: IQueryBuilder<T>, alias?: string): any {
     const nestedAlias = qb.getAliasForJoinPath(this.getPath());
     const ownerAlias = alias || qb.alias;
+    const keys = Object.keys(this.payload);
 
     if (nestedAlias) {
       alias = nestedAlias;
     }
 
     if (this.shouldAutoJoin(nestedAlias)) {
+      if (keys.some(k => ['$some', '$none', '$every'].includes(k))) {
+        const $and: Dictionary[] = [];
+        const primaryKeys = this.metadata.find(this.entityName)!.primaryKeys.map(pk => `${alias}.${pk}`);
+
+        for (const key of keys) {
+          const payload = (this.payload[key] as CriteriaNode<T>).unwrap();
+          const sub = qb.clone(true).innerJoin(this.key!, qb.getNextAlias(this.prop!.type));
+          sub.select(this.prop!.targetMeta!.primaryKeys);
+
+          if (key === '$every') {
+            sub.where({ $not: { [this.key!]: payload } });
+          } else {
+            sub.where({ [this.key!]: payload });
+          }
+
+          const op = key === '$some' ? '$in' : '$nin';
+
+          $and.push({
+            [Utils.getPrimaryKeyHash(primaryKeys)]: { [op]: (sub as Dictionary).getKnexQuery() },
+          });
+        }
+
+        return { $and };
+      }
+
       alias = this.autoJoin(qb, ownerAlias);
     }
 
-    return Object.keys(this.payload).reduce((o, field) => {
+    return keys.reduce((o, field) => {
       const childNode = this.payload[field] as CriteriaNode<T>;
       const payload = childNode.process(qb, this.prop ? alias : ownerAlias);
       const operator = Utils.isOperator(field);
@@ -52,23 +78,32 @@ export class ObjectCriteriaNode<T extends object> extends CriteriaNode<T> {
         o[`${alias}.${field}`] = payload;
       }
 
+
+      return o;
+    }, {} as Dictionary);
+  }
+
+  override unwrap(): any {
+    return Object.keys(this.payload).reduce((o, field) => {
+      o[field] = this.payload[field].unwrap();
       return o;
     }, {} as Dictionary);
   }
 
   override willAutoJoin(qb: IQueryBuilder<T>, alias?: string) {
     const nestedAlias = qb.getAliasForJoinPath(this.getPath());
     const ownerAlias = alias || qb.alias;
+    const keys = Object.keys(this.payload);
 
     if (nestedAlias) {
       alias = nestedAlias;
     }
 
     if (this.shouldAutoJoin(nestedAlias)) {
-      return true;
+      return !keys.some(k => ['$some', '$none', '$every'].includes(k));
     }
 
-    return Object.keys(this.payload).some(field => {
+    return keys.some(field => {
       const childNode = this.payload[field] as CriteriaNode<T>;
       return childNode.willAutoJoin(qb, this.prop ? alias : ownerAlias);
     });
@@ -92,7 +127,8 @@ export class ObjectCriteriaNode<T extends object> extends CriteriaNode<T> {
         o[`${alias}.${field}`] = { [k]: tmp, ...(o[`${alias}.${field}`] || {}) };
       } else if (this.isPrefixed(k) || Utils.isOperator(k) || !childAlias) {
         const idx = prop.referencedPKs.indexOf(k as EntityKey);
-        const key = idx !== -1 && !childAlias ? prop.joinColumns[idx] : k;
+        // FIXME maybe other kinds should be supported too?
+        const key = idx !== -1 && !childAlias && ![ReferenceKind.ONE_TO_MANY, ReferenceKind.MANY_TO_MANY].includes(prop.kind) ? prop.joinColumns[idx] : k;
 
         if (key in o) {
           const $and = o.$and ?? [];
@@ -101,9 +137,7 @@ export class ObjectCriteriaNode<T extends object> extends CriteriaNode<T> {
           o.$and = $and;
         } else if (Utils.isOperator(k) && Array.isArray(payload[k])) {
             o[key] = payload[k].map((child: Dictionary) => Object.keys(child).reduce((o, childKey) => {
-              const key = (this.isPrefixed(childKey) || Utils.isOperator(childKey))
-                ? childKey
-                : `${childAlias}.${childKey}`;
+              const key = (this.isPrefixed(childKey) || Utils.isOperator(childKey)) ? childKey : `${childAlias}.${childKey}`;
               o[key] = child[childKey];
               return o;
             }, {} as Dictionary));
@@ -128,12 +162,15 @@ export class ObjectCriteriaNode<T extends object> extends CriteriaNode<T> {
     const operatorKeys = knownKey && Object.keys(this.payload).every(key => Utils.isOperator(key, false));
     const primaryKeys = knownKey && Object.keys(this.payload).every(key => {
       const meta = this.metadata.find(this.entityName)!;
+
       if (!meta.primaryKeys.includes(key)) {
         return false;
       }
+
       if (!Utils.isPlainObject(this.payload[key].payload) || ![ReferenceKind.ONE_TO_ONE, ReferenceKind.MANY_TO_ONE].includes(meta.properties[key].kind)) {
         return true;
       }
+
       return Object.keys(this.payload[key].payload).every(k => meta.properties[key].targetMeta!.primaryKeys.includes(k));
     });
 

packages/knex/src/query/QueryBuilder.ts

@@ -844,8 +844,13 @@ export class QueryBuilder<T extends object = AnyEntity> {
     return ret;
   }
 
-  clone(): QueryBuilder<T> {
+  clone(reset?: boolean): QueryBuilder<T> {
     const qb = new QueryBuilder<T>(this.mainAlias.entityName, this.metadata, this.driver, this.context, this.mainAlias.aliasName, this.connectionType, this.em);
+
+    if (reset) {
+      return qb;
+    }
+
     Object.assign(qb, this);
 
     // clone array/object properties

packages/knex/src/typings.ts

@@ -137,6 +137,7 @@ export interface IQueryBuilder<T> {
   truncate(): this;
   count(field?: string | string[], distinct?: boolean): this;
   join(field: string, alias: string, cond?: QBFilterQuery, type?: JoinType, path?: string): this;
+  innerJoin(field: string, alias: string, cond?: QBFilterQuery): this;
   leftJoin(field: string, alias: string, cond?: QBFilterQuery): this;
   joinAndSelect(field: string, alias: string, cond?: QBFilterQuery): this;
   leftJoinAndSelect(field: string, alias: string, cond?: QBFilterQuery, fields?: string[]): this;
@@ -153,6 +154,7 @@ export interface IQueryBuilder<T> {
   having(cond?: QBFilterQuery | string, params?: any[]): this;
   getAliasForJoinPath(path: string): string | undefined;
   getNextAlias(entityName?: string): string;
+  clone(reset?: boolean): IQueryBuilder<T>;
 }
 
 export interface ICriteriaNode<T extends object> {