feat: add support for full text searches (#3317)

docs/docs/decorators.md

@@ -181,7 +181,7 @@ See [Defining Entities](defining-entities.md#indexes).
 |--------------|----------|----------|-------------|
 | `name`       | `string` | yes      | index name  |
 | `properties` | `string` | `string[]` | yes | list of properties, required when using on entity level |
-| `type`       | `string` | yes      | index type, not available for `@Unique()` |
+| `type`       | `string` | yes      | index type, not available for `@Unique()`. Use `fulltext` to enable support for the `$fulltext` operator |
 
 ```ts
 @Entity()

docs/docs/entity-manager.md

@@ -136,7 +136,7 @@ const user1 = await em.findOne(User, 1);
 ```
 
 As we can see in the fifth example, one can also use operators like `$and`, `$or`, `$gte`, 
-`$gt`, `$lte`, `$lt`, `$in`, `$nin`, `$eq`, `$ne`, `$like`, `$re`. More about that can be found in 
+`$gt`, `$lte`, `$lt`, `$in`, `$nin`, `$eq`, `$ne`, `$like`, `$re` and `$fulltext`. More about that can be found in 
 [Query Conditions](query-conditions.md) section.
 
 #### Using custom classes in `FilterQuery`

docs/docs/query-conditions.md

@@ -62,6 +62,7 @@ const res = await orm.em.find(Author, [1, 2, 7]);
 | `$nin`   | not contains       | Matches none of the values specified in an array. |
 | `$like`  | like               | Uses LIKE operator |
 | `$re`    | regexp             | Uses REGEXP operator |
+| `$fulltext` | full text       | A driver specific full text search function. See requirements [below](#full-text-searching) |
 | `$ilike` | ilike              | (postgres only) |
 | `$overlap` | &&               | (postgres only) |
 | `$contains` | @>              | (postgres only) |
@@ -74,3 +75,123 @@ const res = await orm.em.find(Author, [1, 2, 7]);
 | `$and`   | Joins query clauses with a logical AND returns all documents that match the conditions of both clauses. |
 | `$not`   | Inverts the effect of a query expression and returns documents that do not match the query expression. |
 | `$or`    | Joins query clauses with a logical OR returns all documents that match the conditions of either clause. |
+
+## Full text searching
+
+Full-text search refers to searching some text inside extensive text data stored and returning results that contain some or all of the words from the query. In contrast, traditional search would return exact matches.
+
+The implementation and requirements differs per driver so it's important that fields are setup correctly.
+
+### PostgreSQL
+
+PosgreSQL allows to execute queries (pg-query) on the type pg-vector. The pg-vector type can be a column (more performant) or be created in the query (no excess columns in the database).
+
+Refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/current/functions-textsearch.html) for possible queries.
+
+<Tabs
+groupId="entity-def"
+defaultValue="as-column"
+values={[
+{label: 'reflect-metadata', value: 'as-column'},
+{label: 'ts-morph', value: 'in-query'},
+]
+}>
+<TabItem value="as-column">
+
+```ts title="./entities/Book.ts"
+import { FullTextType } from '@mikro-orm/postgresql';
+
+@Entity()
+export class Book {
+
+  @Property()
+  title!: string;
+
+  @Index({ type: 'fulltext' })
+  @Property({ type: FullTextType, onUpdate: (book) => book.title })
+  searchableTitle!: string;
+
+}
+```
+
+And to find results: `repository.findOne({ searchableTitle: { $fulltext: 'query' } })`
+
+  </TabItem>
+  <TabItem value="in-query">
+
+```ts title="./entities/Book.ts"
+@Entity()
+export class Book {
+
+  @Index({ type: 'fulltext' })
+  @Property()
+  title!: string;
+
+}
+```
+
+And to find results: `repository.findOne({ title: { $fulltext: 'query' } })`
+
+  </TabItem>
+</Tabs>
+
+### MySQL, MariaDB
+MySQL and MariaDB allow full text searches on all columns with a fulltext index.
+
+Refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/8.0/en/fulltext-boolean.html) or [MariaDB documentation](https://mariadb.com/kb/en/full-text-index-overview/#in-boolean-mode) for possible queries.
+
+```ts title="./entities/Book.ts"
+@Entity()
+export class Book {
+
+  @Index({ type: 'fulltext' })
+  @Property()
+  title!: string;
+
+}
+```
+
+And to find results: `repository.findOne({ title: { $fulltext: 'query' } })`
+
+### MongoDB
+
+MongoDB allows full text searches on all columns with a text index. However, when executing a full text search, it selects matches based on all fields with a text index: it's only possible to add one query and only on the top-level of the query object. Refer to the [MongoDB documentation](https://www.mongodb.com/docs/manual/reference/operator/query/text/#behavior) for more information on this behavior.
+
+Refer to the [MongoDB documentation](https://www.mongodb.com/docs/manual/reference/operator/query/text/#definition) for possible queries.
+
+
+```ts title="./entities/Book.ts"
+@Entity()
+export class Book {
+
+  @Index({ type: 'fulltext' })
+  @Property()
+  title!: string;
+
+}
+```
+
+### SQLite
+
+In SQLite, full text searches can only be executed on [FTS5 virtual tables](https://www.sqlite.org/fts5.html#overview_of_fts5). MikroORM can't create this table, and has to be done [manually](https://www.sqlite.org/fts5.html#fts5_table_creation_and_initialization). Simple tables can be created with this query:
+
+`CREATE VIRTUAL TABLE <table name> USING fts5(<colum1>, <column2>, ...);`
+
+Afterwards an entity can created normally for the structure of this table. The `@Index` is not neccessary for full text searches in SQLite.
+
+Refer to the [SQLite documentation](https://www.sqlite.org/fts5.html#full_text_query_syntax) for possible queries.
+
+```ts title="./entities/Book.ts"
+@Entity()
+export class Book {
+
+  @PrimaryKey()
+  id!: number;
+
+  @Property()
+  title!: string;
+
+}
+```
+
+And to find results: `repository.findOne({ title: { $fulltext: 'query' } })`

packages/core/src/enums.ts

@@ -32,6 +32,7 @@ export enum QueryOperator {
   $not = 'not',
   $like = 'like',
   $re = 'regexp',
+  $fulltext = 'fulltext',
   $exists = 'not null',
   $ilike = 'ilike', // postgres only
   $overlap = '&&', // postgres only

packages/core/src/index.ts

@@ -7,7 +7,7 @@ export {
   Constructor, ConnectionType, Dictionary, PrimaryKeyType, PrimaryKeyProp, Primary, IPrimaryKey, ObjectQuery, FilterQuery, IWrappedEntity, EntityName, EntityData, Highlighter,
   AnyEntity, EntityClass, EntityProperty, EntityMetadata, QBFilterQuery, PopulateOptions, Populate, Loaded, New, LoadedReference, LoadedCollection, IMigrator, IMigrationGenerator,
   GetRepository, EntityRepositoryType, MigrationObject, DeepPartial, PrimaryProperty, Cast, IsUnknown, EntityDictionary, EntityDTO, MigrationDiff,
-  IEntityGenerator, ISeedManager, EntityClassGroup, OptionalProps, RequiredEntityData, CheckCallback,
+  IEntityGenerator, ISeedManager, EntityClassGroup, OptionalProps, RequiredEntityData, CheckCallback, SimpleColumnMeta,
 } from './typings';
 export * from './enums';
 export * from './errors';

packages/core/src/platforms/Platform.ts

@@ -2,7 +2,7 @@ import { clone } from '../utils/clone';
 import { EntityRepository } from '../entity';
 import type { NamingStrategy } from '../naming-strategy';
 import { UnderscoreNamingStrategy } from '../naming-strategy';
-import type { AnyEntity, Constructor, EntityProperty, IEntityGenerator, IMigrator, IPrimaryKey, ISchemaGenerator, PopulateOptions, Primary, EntityMetadata } from '../typings';
+import type { AnyEntity, Constructor, EntityProperty, IEntityGenerator, IMigrator, IPrimaryKey, ISchemaGenerator, PopulateOptions, Primary, EntityMetadata, SimpleColumnMeta } from '../typings';
 import { ExceptionConverter } from './ExceptionConverter';
 import type { EntityManager } from '../EntityManager';
 import type { Configuration } from '../utils/Configuration';
@@ -129,6 +129,10 @@ export abstract class Platform {
     return 'regexp';
   }
 
+  isAllowedTopLevelOperator(operator: string) {
+    return operator === '$not';
+  }
+
   quoteVersionValue(value: Date | number, prop: EntityProperty): Date | string | number {
     return value;
   }
@@ -291,6 +295,18 @@ export abstract class Platform {
     return path.join('.');
   }
 
+  getFullTextWhereClause(prop: EntityProperty<any>): string {
+    throw new Error('Full text searching is not supported by this driver.');
+  }
+
+  supportsCreatingFullTextIndex(): boolean {
+    throw new Error('Full text searching is not supported by this driver.');
+  }
+
+  getFullTextIndexExpression(indexName: string, schemaName: string | undefined, tableName: string, columns: SimpleColumnMeta[]): string {
+    throw new Error('Full text searching is not supported by this driver.');
+  }
+
   convertsJsonAutomatically(marshall = false): boolean {
     return !marshall;
   }

packages/core/src/typings.ts

@@ -69,6 +69,7 @@ export type OperatorMap<T> = {
   $like?: string;
   $re?: string;
   $ilike?: string;
+  $fulltext?: string;
   $overlap?: string[];
   $contains?: string[];
   $contained?: string[];
@@ -420,6 +421,11 @@ export class EntityMetadata<T extends AnyEntity<T> = any> {
 
 }
 
+export interface SimpleColumnMeta {
+  name: string;
+  type: string;
+}
+
 export interface EntityMetadata<T extends AnyEntity<T> = any> {
   name?: string; // abstract classes do not have a name, but once discovery ends, we have only non-abstract classes stored
   className: string;

packages/core/src/utils/QueryHelper.ts

@@ -127,8 +127,8 @@ export class QueryHelper {
         return o;
       }
 
-      // wrap top level operators (except $not) with PK
-      if (Utils.isOperator(key) && root && meta && key !== '$not') {
+      // wrap top level operators (except platform allowed operators) with PK
+      if (Utils.isOperator(key) && root && meta && !options.platform.isAllowedTopLevelOperator(key)) {
         const rootPrimaryKey = Utils.getPrimaryKeyHash(meta.primaryKeys);
         o[rootPrimaryKey] = { [key]: QueryHelper.processWhere<T>({ ...options, where: value, root: false }) };
         return o;

packages/core/src/utils/Utils.ts

@@ -822,6 +822,22 @@ export class Utils {
     return !!GroupOperator[key];
   }
 
+  static hasNestedKey(object: unknown, key: string): boolean {
+    if (!object) {
+      return false;
+    }
+
+    if (Array.isArray(object)) {
+      return object.some(o => this.hasNestedKey(o, key));
+    }
+
+    if (typeof object === 'object') {
+      return Object.entries(object).some(([k, v]) => k === key || this.hasNestedKey(v, key));
+    }
+
+    return false;
+  }
+
   static getGlobalStorage(namespace: string): Dictionary {
     const key = `mikro-orm-${namespace}`;
     global[key] = global[key] || {};

packages/knex/src/query/QueryBuilderHelper.ts

@@ -377,7 +377,17 @@ export class QueryBuilderHelper {
       return void qb[m](this.knex.raw(`(${this.subQueries[key]})`), replacement, value[op]);
     }
 
-    qb[m](this.mapper(key, type, undefined, null), replacement, value[op]);
+    if (op === '$fulltext') {
+      const meta = this.metadata.get(this.entityName);
+      const columnName = key.includes('.') ?  key.split('.')[1] : key;
+
+      qb[m](this.knex.raw(this.platform.getFullTextWhereClause(meta.properties[columnName]), {
+        column: this.mapper(key, type, undefined, null),
+        query: value[op],
+      }));
+    } else {
+      qb[m](this.mapper(key, type, undefined, null), replacement, value[op]);
+    }
   }
 
   private getOperatorReplacement(op: string, value: Dictionary): string {

packages/knex/src/schema/SchemaComparator.ts

@@ -130,6 +130,7 @@ export class SchemaComparator {
       renamedColumns: {},
       renamedIndexes: {},
       fromTable,
+      toTable,
     };
 
     if (this.diffComment(fromTable.comment, toTable.comment)) {
@@ -463,8 +464,8 @@ export class SchemaComparator {
    * Compares index1 with index2 and returns index2 if there are any differences or false in case there are no differences.
    */
   diffIndex(index1: Index, index2: Index): boolean {
-    // if one of them is a custom expression, compare only by name
-    if (index1.expression || index2.expression) {
+    // if one of them is a custom expression or full text index, compare only by name
+    if (index1.expression || index2.expression || index1.type === 'fulltext' || index2.type === 'fulltext') {
       return index1.keyName !== index2.keyName;
     }
 

packages/knex/src/schema/SchemaGenerator.ts

@@ -390,17 +390,17 @@ export class SchemaGenerator extends AbstractSchemaGenerator<AbstractSqlDriver>
       }
 
       for (const index of Object.values(diff.addedIndexes)) {
-        this.createIndex(table, index, diff.fromTable);
+        this.createIndex(table, index, diff.toTable);
       }
 
       for (const index of Object.values(diff.changedIndexes)) {
-        this.createIndex(table, index, diff.fromTable, true);
+        this.createIndex(table, index, diff.toTable, true);
       }
 
       for (const [oldIndexName, index] of Object.entries(diff.renamedIndexes)) {
         if (index.unique) {
           this.dropIndex(table, index, oldIndexName);
-          this.createIndex(table, index, diff.fromTable);
+          this.createIndex(table, index, diff.toTable);
         } else {
           this.helper.pushTableQuery(table, this.helper.getRenameIndexSQL(diff.name, index, oldIndexName));
         }
@@ -514,6 +514,12 @@ export class SchemaGenerator extends AbstractSchemaGenerator<AbstractSqlDriver>
       table.unique(index.columnNames, { indexName: index.keyName });
     } else if (index.expression) {
       this.helper.pushTableQuery(table, index.expression);
+    } else if (index.type === 'fulltext') {
+      const columns = index.columnNames.map(name => ({ name, type: tableDef.getColumn(name)!.type }));
+
+      if (this.platform.supportsCreatingFullTextIndex()) {
+        this.helper.pushTableQuery(table, this.platform.getFullTextIndexExpression(index.keyName, tableDef.schema, tableDef.name, columns));
+      }
     } else {
       table.index(index.columnNames, index.keyName, index.type as Dictionary);
     }

packages/knex/src/typings.ts

@@ -87,6 +87,7 @@ export interface TableDifference {
   name: string;
   changedComment?: string;
   fromTable: DatabaseTable;
+  toTable: DatabaseTable;
   addedColumns: Dictionary<Column>;
   changedColumns: Dictionary<ColumnDifference>;
   removedColumns: Dictionary<Column>;

packages/mariadb/src/MariaDbPlatform.ts

@@ -1,7 +1,7 @@
 import { AbstractSqlPlatform } from '@mikro-orm/knex';
 import { MariaDbSchemaHelper } from './MariaDbSchemaHelper';
 import { MariaDbExceptionConverter } from './MariaDbExceptionConverter';
-import type { Type } from '@mikro-orm/core';
+import type { SimpleColumnMeta, Type } from '@mikro-orm/core';
 import { expr, Utils } from '@mikro-orm/core';
 
 export class MariaDbPlatform extends AbstractSqlPlatform {
@@ -69,4 +69,20 @@ export class MariaDbPlatform extends AbstractSqlPlatform {
     return 'PRIMARY'; // https://dev.mysql.com/doc/refman/8.0/en/create-table.html#create-table-indexes-keys
   }
 
+  supportsCreatingFullTextIndex(): boolean {
+    return true;
+  }
+
+  getFullTextWhereClause(): string {
+    return `match(:column:) against (:query in boolean mode)`;
+  }
+
+  getFullTextIndexExpression(indexName: string, schemaName: string | undefined, tableName: string, columns: SimpleColumnMeta[]): string {
+    const quotedTableName = this.quoteIdentifier(schemaName ? `${schemaName}.${tableName}` : tableName);
+    const quotedColumnNames = columns.map(c => this.quoteIdentifier(c.name));
+    const quotedIndexName = this.quoteIdentifier(indexName);
+
+    return `alter table ${quotedTableName} add fulltext index ${quotedIndexName}(${quotedColumnNames.join(',')})`;
+  }
+
 }