Skip to content

Commit

Permalink
Highlighting in SearchControl
Browse files Browse the repository at this point in the history
  • Loading branch information
@olmobrutall
olmobrutall committed Apr 18, 2023
1 parent e970549 commit fbba1e4
Show file tree
Hide file tree
Showing 18 changed files with 364 additions and 133 deletions.
1 change: 1 addition & 0 deletions Signum.Entities/DynamicQuery/Filter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,7 @@ public static List<FilterFullText> TableFilters(List<Filter> filters)
return filters.SelectMany(a => a.GetAllFilters()).OfType<FilterFullText>().Where(a => a.IsTable).ToList();
}

//Keep in sync with Finder.tsx extractComplexConditions
public override IEnumerable<string> GetKeywords()
{
if (this.Operation == FullTextFilterOperation.FreeText)
Expand Down
2 changes: 1 addition & 1 deletion Signum.Entities/DynamicQuery/Tokens/EntityPropertyToken.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ protected override List<QueryToken> SubTokensOverride(SubTokensOptions options)
result.Add(new FullTextRankToken(this));
}

if (this.HasSnippet)
if (this.HasSnippet && (options & SubTokensOptions.CanToArray) != 0)
{
result.Add(new StringSnippetToken(this));
}
Expand Down
1 change: 1 addition & 0 deletions Signum.Entities/EnumMessages.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ public enum SearchMessage
[Description("Delete all filter")]
DeleteAllFilter,
Filters,
Columns,
Find,
[Description("Finder of {0}")]
FinderOf0,
Expand Down
8 changes: 4 additions & 4 deletions Signum.React.Extensions/Authorization/AuthAdminClient.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import * as React from 'react'
import { RouteObject } from 'react-router'
import { ModifiableEntity, EntityPack, is, OperationSymbol, SearchMessage, Lite, getToString, EntityControlMessage } from '@framework/Signum.Entities';
import { ModifiableEntity, EntityPack, is, OperationSymbol, SearchMessage, Lite, getToString, EntityControlMessage, liteKeyLong } from '@framework/Signum.Entities';
import { ifError } from '@framework/Globals';
import { ajaxPost, ajaxGet, ajaxGetRaw, saveFile, ServiceError } from '@framework/Services';
import * as Services from '@framework/Services';
Expand Down Expand Up @@ -41,14 +41,14 @@ export function start(options: { routes: RouteObject[], types: boolean; properti
permissions = options.permissions;

Navigator.addSettings(new EntitySettings(UserEntity, e => import('./Templates/User'), {
renderLite: (lite, subStr) => {
renderLite: (lite, hl) => {
if (UserLiteModel.isInstance(lite.model))
return (
<span className="d-inline-flex align-items-center"><SmallProfilePhoto user={lite} className="me-1" /><span>{TypeaheadOptions.highlightedText(getToString(lite), subStr)}</span></span>
<span className="d-inline-flex align-items-center"><SmallProfilePhoto user={lite} className="me-1" /><span>{hl.highlight(getToString(lite))}</span></span>
);

if (typeof lite.model == "string")
return TypeaheadOptions.highlightedText(getToString(lite), subStr);
return hl.highlight(getToString(lite));

return lite.EntityType;
}
Expand Down
6 changes: 3 additions & 3 deletions Signum.React.Extensions/Basics/Templates/IconTypeahead.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { TypeContext } from '@framework/TypeContext'
import { library } from '@fortawesome/fontawesome-svg-core'
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'
import { useForceUpdate } from '@framework/Hooks'
import { TypeaheadOptions } from '@framework/Components/Typeahead'
import { TextHighlighter, TypeaheadOptions } from '@framework/Components/Typeahead'
import { IconName, IconProp, IconPrefix } from "@fortawesome/fontawesome-svg-core";

export interface IconTypeaheadLineProps {
Expand Down Expand Up @@ -99,13 +99,13 @@ export function IconTypeahead(p: IconTypeaheadProps) {
return item as string;
}

function handleRenderItem(item: unknown, query: string) {
function handleRenderItem(item: unknown, hl: TextHighlighter) {
var icon = parseIcon(item as string);

return (
<span>
{icon && <FontAwesomeIcon icon={icon} className="icon" style={{ width: "12px", height: "12px" }} />}
{TypeaheadOptions.highlightedTextAll(item as string, query)}
{hl.highlight(item as string)}
</span>
);
}
Expand Down
18 changes: 18 additions & 0 deletions Signum.React/Scripts/AppContext.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ declare global {

interface Array<T> {
joinCommaHtml(this: Array<T>, lastSeparator: string): React.ReactElement<any>;
joinHtml(this: Array<T>, separator: string | React.ReactElement<any>): React.ReactElement<any>;
}
}

Expand Down Expand Up @@ -196,3 +197,20 @@ Array.prototype.joinCommaHtml = function (this: any[], lastSeparator: string) {
return React.createElement("span", undefined, ...result);
}

Array.prototype.joinHtml = function (this: any[], separator: string | React.ReactElement<any>) {
const args = arguments;

const result: (string | React.ReactElement<any>)[] = [];
for (let i = 0; i < this.length -1; i++) {
result.push(this[i]);
result.push(separator);
}


if (this.length >= 1) {
result.push(this[this.length - 1]);
}

return React.createElement("span", undefined, ...result);
}

80 changes: 40 additions & 40 deletions Signum.React/Scripts/Components/Typeahead.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ export interface TypeaheadProps {
itemsDelay?: number;
minLength?: number;
renderList?: (typeahead: TypeaheadController) => React.ReactNode;
renderItem?: (item: unknown, query: string) => React.ReactNode;
renderItem?: (item: unknown, highlighter: TextHighlighter) => React.ReactNode;
onSelect?: (item: unknown, e: React.KeyboardEvent<any> | React.MouseEvent<any>) => string | null;
scrollHeight?: number;
inputAttrs?: React.InputHTMLAttributes<HTMLInputElement>;
Expand Down Expand Up @@ -257,6 +257,8 @@ export const Typeahead = React.forwardRef(function Typeahead(p: TypeaheadProps,

function renderDefaultList() {
var items = controller.items;

var highlighter = TextHighlighter.fromString(controller.query);
return (
<Dropdown.Menu align={controller.rtl ? "end" : undefined} className="typeahead">
{
Expand All @@ -268,7 +270,7 @@ export const Typeahead = React.forwardRef(function Typeahead(p: TypeaheadProps,
onMouseLeave={e => controller.handleElementMouseLeave(e, i)}
onMouseUp={e => controller.handleMenuMouseUp(e, i)}
{...p.itemAttrs && p.itemAttrs(item)}>
{p.renderItem!(item, controller.query!)}
{p.renderItem!(item, highlighter)}
</button>)
}
</Dropdown.Menu>
Expand All @@ -293,7 +295,7 @@ Typeahead.defaultProps = {
getItems: undefined as any,
itemsDelay: 200,
minLength: 1,
renderItem: (item, query) => TypeaheadOptions.highlightedText(item as string, query),
renderItem: (item, highlighter) => highlighter.highlight(item as string),
onSelect: (elem, event) => (elem as string),
scrollHeight: 0,

Expand All @@ -302,57 +304,55 @@ Typeahead.defaultProps = {


export namespace TypeaheadOptions {
export function highlightedText(val: string, query?: string): React.ReactChild {

if (query == undefined)
return val;
export function normalizeString(str: string): string {
return str;
}
}

const index = val.toLowerCase().indexOf(query.toLowerCase());
if (index == -1)
return val;
export class TextHighlighter {
query?: string;
parts?: string[];
regex?: RegExp;

return (
<>
{val.substr(0, index)}
<strong key={0}>{val.substr(index, query.length)}</strong>
{val.substr(index + query.length)}
</>
);
static fromString(query: string | undefined) {
var hl = new TextHighlighter(query?.split(" "));
hl.query = query;
return hl;
}

export function highlightedTextAll(val: string, query: string | undefined): React.ReactChild {
if (query == undefined)
return val;
constructor(parts: string[] | undefined) {
this.parts = parts?.filter(a => a != null && a.length > 0).orderByDescending(a => a.length);
if (this.parts?.length)
this.regex = new RegExp(this.parts.map(p => RegExp.escape(p)).join("|"), "gi");
}

const parts = query.toLocaleLowerCase().split(" ").filter(a => a.length > 0).orderByDescending(a => a.length);
highlight(text: string): React.ReactChild {
if (!text || !this.regex)
return text;

function splitText(str: string, partIndex: number): React.ReactChild {
var matches = Array.from(text.matchAll(this.regex));

if (str.length == 0)
return str;
if (matches.length == 0)
return text;

if (parts.length <= partIndex)
return str;
var result = [];

var part = parts[partIndex];
var pos = 0;
for (var i = 0; i < matches.length; i++) {
var m = matches[i];

const index = str.toLowerCase().indexOf(part);
if (index == -1)
return splitText(str, partIndex + 1);
if (pos < m.index!) {
result.push(text.substring(pos, m.index));
}

return (
<>
{splitText(str.substr(0, index), partIndex + 1)}
<strong key={0}>{str.substr(index, part.length)}</strong>
{splitText(str.substr(index + part.length), partIndex + 1)}
</>
);
pos = m.index! + m[0].length;
result.push(<strong>{text.substring(m.index!, pos)}</strong>);
}

return splitText(val, 0);
}
if (pos < text.length)
result.push(text.substring(pos));

export function normalizeString(str: string): string {
return str;
return React.createElement(React.Fragment, undefined, ...result);
}
}

2 comments on commit fbba1e4

@olmobrutall
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Full-Text Search, Snippets and Highlight in SearchControl

In this post I want to present three unrelated features that complement each other:

  • Full-Text Search (in LINQ Provider and SearchControl)
  • Snippets in SearchControl
  • Hightlight in SearchControl
  • SplitValue and EntityStrip in SearchControl

All three are meant to improve the experience of searching by text, either by making it faster, or showing the results in a more user-friendly way.

What is Full-Text Search.

Full-Text Search is a relatevly mature feature of SQL Server that has taken way too long to come into Signum Framework, and it can be a good alternative to more heavyweight solutions like ElasticSearch for most of the cases.

The idea is to index all the words found in a document, and when a query is made, it will be splitted into words and search the document where this words are found.

This makes the query much faster but also means that, by default, searching by "Signu" won't find results for "Signum" because they are different words.

This index is called Full-Text index and requires:

  • One or more string columns to index
  • A unique column (typically the ID) to reference the results.

There is only one Full-Text index per table and all the selected string columns of the table will be stored together in the same Full-Text index. You can imagine as a table with the following structure.

  • Word: The word to search for (Signum)
  • Key: The Unique ID of the row (4)
  • ColumnIndex: The Column where the word is found.
  • Position: The position inside of the column text.

The topic is explain in more depth here, but let me do a small explanation of some tricky concepts.

Once you have a Full-Text Index, SQL Server allows you to query it using two functions: CONTAINS and FREETEXT. Both functions return a boolean.

  • CONTAINS(columns, searchCondition): Returns True if the column or columns match the searchCondition.
    • columns should be one or more expressions referencing a database column, with implicit or explicit table alias (p.Name or just Name). Only columns of the same table are allowed, and you search in all the full-text indexed columns using p.*. If more than one column is used, use parenthesys like in a tuple: CONTAINS((p.Notes, p.Qualifications), "C#")
    • searchCondition string containing a mini-SDL to define the right match, allowing boolean operators and quotes for words that should appear in the same order (("Signum Framework" OR LINQ OR TypeScript OR JavaSCript) AND NOT Microservice).

Example:

SELECT p.Name 
 WHERE CONTAINS(p.Remarks, 'C# OR Java`)
 FROM dbo.Person p
  • FREETEXT(columns, freeText): Returns True if the column or columns have some fuzzy similarity with the text written in freeText.
    • columns similar to CONTAINS columns
    • freeText unestructured sentence that should be similar to the results. Example: `"Signum Framework TypeScript"

Example:

SELECT p.Name 
 WHERE FREETEXT(p.Remarks, 'Signum Framework C# TypeScript'`)
 FROM dbo.Person p

This two function have the limitation that, by returning a boolean, do not allow ranking the results by better matching, that's why another two functions are needed: CONTAINSTABLE and FREETEXTTABLE.

The -TABLE variants allow you direct access to the full-text index, returning a view.

  • CONTAINSTABLE(table, columns, searchCondition): Returns a view with RANK and KEY. Key is the unique column (the ID) while RANK is a value that is bigger the better the result is.
    • table: The name of the table containing the full-text index (dbo.Person)
    • columns: The column of columns (in a tuple). Use * for all columns. Note that this are column names defined in table, not expression referencing a column with an optional alias.
    • searchCondition Identical to CONTAINS searchCondition.
    • top_n_by_rank: Optional, limits to N results
# Using CONTAINSTABLE alone
SELECT r.KEY, r.RANK
FROM CONTAINSTABLE(dbo.Person, Notes,  'C# OR Java', 10) as r

# Using CONTAINSTABLE with JOIN
SELECT p.Name, r.Rank
FROM CONTAINSTABLE(dbo.Person, Notes,  '"Signum Framework" OR C# OR TypeScript') as r
JOIN dbo.Person p on r.Key = p.Id
ORDER BY r.Rank descending
  • FREETEXTTABLE(table, columns, searchCondition): Similar to CONTAINSTABLE but with freeText instead of searchCondition.
# Using FREETEXTTABLE alone
SELECT r.KEY, r.RANK
FROM FREETEXTTABLE(dbo.Person, Notes,  'C# OR Java', 10) as r

# Using FREETEXTTABLE with JOIN
SELECT p.Name, r.Rank
FROM FREETEXTTABLE(dbo.Person, Notes,  'Signum Framework C# TypeScript') as r
JOIN dbo.Person p on r.Key = p.Id
ORDER BY r.Rank descending

Registering a Full-Text Search Index

Registering is as simple as using WithFullTextIndex:

sb.Include<EmployeeEntity>()
	.WithSave(EmployeeOperation.Save)
	.WithFullTextIndex(e => new { e.FirstName, e.LastName, e.Notes })
	.WithQuery(() => e => new
	{
		Entity = e,
		e.Id,
		e.FirstName,
		e.LastName,
		e.BirthDate,
		e.Photo, //1
	});

Since Full-Text Index is not supported by default in SQL Server Express, maybe you want to include it conditionally:

if (Connector.Current.SupportsFullTextSearch)
{
	sb.AddFullTextIndex((EmployeeEntity e) => new { e.FirstName, e.LastName, e.Notes });
}

The Synchronizer / SQL Migrations will take care of creating the Full-Text index and the Full-Text catallog if necessary (can not be created transactionally!).

Full-Text Functions in LINQ Provider

The LINQ Provider contains methods that mimic the SQL Server counterparts:

For example:

SELECT p.Name 
 WHERE CONTAINS(p.Remarks, 'C# OR Java`)
 FROM dbo.Person p

Is equivalent to:

var res = (from p in Database.Query<PersonEntity>()
           where FullTextSearch.Contains(new[] { p.Remarks }, "C# OR Java")
           select p.Name).ToList();

And:

SELECT p.Name, r.Rank
FROM FREETEXTTABLE(dbo.Person, Notes,  'Signum Framework C# TypeScript') as r
JOIN dbo.Person p on r.Key = p.Id
ORDER BY p.Rank descending
var res = 
   (from r in Database.Query<PersonEntity>()
    join ft in FullTextSearch.FreeTextTable<PersonEntity>(p=> p.Notes, "Signum Framework C# TypeScript")
    on r.Id equals ft.Key
    orderby ft.Rank descending
    select new { Lite = r.ToLite(), ft.Rank }
    ).ToList();

You can find more examples in Signum.Test FullTextSearchTest.cs

Full-Text Operators in SearchControl Filters

Knowing how the integration of Full-Text in the LINQ Provider works is usefull if you are building a custom Product Catallog or something like this,
but in Sigum apps most of the queries are made through the Search Control, so we have also build a firs-class integration.

Whenever a columns has a full-text index, two new filter operations are available: ComplexCondition (related to CONTAINS SQL method) and FreeText.

image

If you combine multiple CONTAINS for different columns of the same table in an OR group with a pinned filter, the query will be simplified.

image

So instead of

CONTAINS(p.FirstName, "Nancy AND Colorado") OR CONTAINS(p.FirstName, "Nancy AND Colorado") OR CONTAINS(p.Notes, "Nancy AND Colorado")

The query will automatically be:

CONTAINS((p.FirsNAme, p.LastName, p.Notes), "Nancy AND Colorado")

Additionally, you can add a new QueryToken to columns or orders called MatchRank, available for every column in a full-text search.

image

Whenever this token is found as a column and/or as an order, then the ComplexCondition and FreeText filters will be promoted to use CONTAINSTABLE / FREETEXTTABLE instead, and the rank returned will be used for this column or order.

Text Snippet in SearchControl

Full-Text search is useful for columns with long texts (NVARCHART(MAX)), but you don't want to spit all the content to the user in the SearchControl.

For this reasons by default any string column with more than 200 charactes (with or without full-text search) get an additional MatchSnippet query token that will return a Google-Like summary of the document, only showing the sentences where the macthing words are found.

image

The algorithm understands keywords using the new filter operations (ComplexCondition and FreeText) as well as the traditional ones (Contains, StartsWith, etc..) and is executed in the server side.

As a bonus, ordering by a MatchSnippet (ascending) is automatically transformed to ordering by a MatchRank (descending).

Highlight in SearchControl

Finally, to improve the UI even more, the search terms are now highlighted in bold in the search results.

This feature is avaiable for any string column, not only the big ones, so I expect it to be the more popular one.

Also is able to detect filters with exactly the same token (Name), or similar tokens with the same property route (Entity.Name vs Name) and understands the ToStringExpresion (Name vs [ToStr]).

The highlight algorithm also understands keywords using the new filter operations (ComplexCondition and FreeText) as well as the traditional ones (Contains, StartsWith, etc..) but this time is executed in the client side.

You can highlight you custom CellFormatter / renderLite using the new TextHighlighter class.

Example from formatRules in Finder:

{
	name: "Object",
	isApplicable: qt => true,
	formatter: (qt, sc) => {
		var hl = new TextHighlighter(getKeywordsSC(qt, sc));
		return new CellFormatter(cell => cell ? <span className="try-no-wrap">{hl.highlight(cell.toString())}</span> : undefined, true);
	}
},

Conclusion

The SearchControl continues to be a flexible component for deterministic filtering of relational data, but with the new Full-Text Search operators, Text Snippets, and Text Hightlights now it has gained some super powers for building a more google-like UI, where all the search is text based, and the results need to be ranked.

Enjoy!

@mehdy-karimpour
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Excellent, very useful feature 🥇

Please sign in to comment.