All: Fix search filters when language is in Korean or with accents (#…

…3947)

ReactNativeClient/lib/services/searchengine/SearchEngine.js

@@ -577,6 +577,22 @@ class SearchEngine {
 			keys.push(col);
 		}
 
+		//
+		// The object "allTerms" is used for query construction purposes (this contains all the filter terms)
+		// Since this is used for the FTS match query, we need to normalize text, title and body terms.
+		// Note, we're not normalizing terms like tag because these are matched using SQL LIKE operator and so we must preserve their diacritics.
+		//
+		// The object "terms" only include text, title, body terms and is used for highlighting.
+		// By not normalizing the text, title, body in "terms", highlighting still works correctly for words with diacritics.
+		//
+
+		allTerms = allTerms.map(x => {
+			if (x.name === 'text' || x.name === 'title' || x.name === 'body') {
+				return Object.assign(x, { value: this.normalizeText_(x.value) });
+			}
+			return x;
+		});
+
 		return {
 			termCount: termCount,
 			keys: keys,
@@ -635,7 +651,15 @@ class SearchEngine {
 		// If preferredSearchType is "fts" we auto-detect anyway
 		// because it's not always supported.
 
-		const st = scriptType(query);
+		let allTerms = [];
+		try {
+			allTerms = filterParser(query);
+		} catch (error) {
+			console.warn(error);
+		}
+
+		const textQuery = allTerms.filter(x => x.name === 'text' || x.name == 'title' || x.name == 'body').map(x => x.value).join(' ');
+		const st = scriptType(textQuery);
 
 		if (!Setting.value('db.ftsEnabled') || ['ja', 'zh', 'ko', 'th'].indexOf(st) >= 0) {
 			return SearchEngine.SEARCH_TYPE_BASIC;
@@ -653,12 +677,11 @@ class SearchEngine {
 			fuzzy: Setting.value('db.fuzzySearchEnabled') === 1,
 		}, options);
 
-		searchString = this.normalizeText_(searchString);
-
 		const searchType = this.determineSearchType_(searchString, options);
 
 		if (searchType === SearchEngine.SEARCH_TYPE_BASIC) {
 			// Non-alphabetical languages aren't support by SQLite FTS (except with extensions which are not available in all platforms)
+			searchString = this.normalizeText_(searchString);
 			const rows = await this.basicSearch(searchString);
 			const parsedQuery = await this.parseQuery(searchString);
 			this.processResults_(rows, parsedQuery, true);