feat(query): fix incorrect matching of Thai characters in FTS5 (#216)

pelias · Dec 13, 2021 · a60df73 · a60df73
1 parent 495b74f
commit a60df73
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 7 deletions.
diff --git a/lib/Queries.js b/lib/Queries.js
@@ -152,10 +152,12 @@ module.exports.matchSubjectObjectGeomIntersects = function( subject, object, cb
     if( '' === object.trim() ){ return cb( null, [] ); }
 
     this._queryAll(
-      this.prepare( query.match_subject_object_geom_intersects ),
+      this.prepare( query.match_subject_object_geom_intersects_autocomplete ),
       {
-        subject: `"${subject}"`,
-        object: `"${object}" OR "${object}"*`,
+        subject,
+        object,
+        subject_quoted: `"${subject}"`,
+        object_quoted: `"${object}"`,
         threshold: RTREE_THRESHOLD,
         limit: MAX_RESULTS
       },
@@ -165,8 +167,10 @@ module.exports.matchSubjectObjectGeomIntersects = function( subject, object, cb
     this._queryAll(
       this.prepare( query.match_subject_object_geom_intersects ),
       {
-        subject: `"${subject}"`,
-        object: `"${object}"`,
+        subject,
+        object,
+        subject_quoted: `"${subject}"`,
+        object_quoted: `"${object}"`,
         threshold: RTREE_THRESHOLD,
         limit: MAX_RESULTS
       },

diff --git a/query/match_subject_object_geom_intersects.sql b/query/match_subject_object_geom_intersects.sql
@@ -11,17 +11,19 @@ FROM fulltext f1
         (r1.minY - $threshold) < r2.maxY AND
         (r1.maxY + $threshold) > r2.minY
       )
-        JOIN fulltext AS f2 ON f2.fulltext MATCH $object
+        JOIN fulltext AS f2 ON f2.fulltext MATCH $object_quoted
           JOIN tokens t2 ON (
             f2.rowid = t2.rowid
             AND r2.id = t2.id
+            AND t2.token = $object
             AND (
               t1.lang = t2.lang OR
               t1.lang IN ('eng', 'und') OR
               t2.lang IN ('eng', 'und')
             )
           )
-WHERE f1.fulltext MATCH $subject
+WHERE f1.fulltext MATCH $subject_quoted
+AND t1.token = $subject
 GROUP BY t1.id, t2.id
 ORDER BY t1.id ASC, t2.id ASC
 LIMIT $limit
diff --git a/query/match_subject_object_geom_intersects_autocomplete.sql b/query/match_subject_object_geom_intersects_autocomplete.sql
@@ -0,0 +1,29 @@
+SELECT
+  t1.id AS subjectId,
+  t2.id as objectId
+FROM fulltext f1
+  JOIN tokens t1 ON f1.rowid = t1.rowid
+    JOIN rtree AS r1 ON t1.id = r1.id
+      JOIN rtree AS r2 ON (
+        r1.maxZ < r2.minZ AND
+        (r1.minX - $threshold) < r2.maxX AND
+        (r1.maxX + $threshold) > r2.minX AND
+        (r1.minY - $threshold) < r2.maxY AND
+        (r1.maxY + $threshold) > r2.minY
+      )
+        JOIN fulltext AS f2 ON f2.fulltext MATCH $object_quoted OR $object_quoted*
+          JOIN tokens t2 ON (
+            f2.rowid = t2.rowid
+            AND r2.id = t2.id
+            AND (t2.token = $object OR t2.token LIKE ($object || '%'))
+            AND (
+              t1.lang = t2.lang OR
+              t1.lang IN ('eng', 'und') OR
+              t2.lang IN ('eng', 'und')
+            )
+          )
+WHERE f1.fulltext MATCH $subject_quoted
+AND t1.token = $subject
+GROUP BY t1.id, t2.id
+ORDER BY t1.id ASC, t2.id ASC
+LIMIT $limit