fix(custom_multiword_synonyms): re-enable support for custom multi-wo…

…rd synonyms
pelias · Aug 4, 2020 · 19e5650 · 19e5650
1 parent e1ce956
commit 19e5650
Show file tree

Hide file tree

Showing 4 changed files with 82 additions and 2 deletions.
diff --git a/settings.js b/settings.js
@@ -34,6 +34,7 @@ function generate(){
           "filter": [
             "lowercase",
             "trim",
+            "synonyms/custom_admin/multiword",
             "admin_synonyms_multiplexer",
             "icu_folding",
             "word_delimiter",
@@ -49,6 +50,7 @@ function generate(){
           "filter": [
             "lowercase",
             "trim",
+            "synonyms/custom_name/multiword",
             "name_synonyms_multiplexer",
             "icu_folding",
             "remove_ordinals",
@@ -81,6 +83,7 @@ function generate(){
             "lowercase",
             "trim",
             "remove_duplicate_spaces",
+            "synonyms/custom_name/multiword",
             "name_synonyms_multiplexer",
             "icu_folding",
             "remove_ordinals",
@@ -126,6 +129,7 @@ function generate(){
             "lowercase",
             "trim",
             "remove_duplicate_spaces",
+            "synonyms/custom_street/multiword",
             "street_synonyms_multiplexer",
             "icu_folding",
             "remove_ordinals",
@@ -227,10 +231,23 @@ function generate(){
   // dynamically create filters for all synonym files in the ./synonyms directory.
   // each filter is given the same name as the file, paths separators are replaced with
   // underscores and the file extension is removed.
-  _.each(synonyms, (synonym, name) => {
+  // note: if no synonym entries are present in the list we use an array
+  // containing an empty space to avoid elasticsearch schema parsing errors.
+  _.each(synonyms, (entries, name) => {
+
+    const singleWordEntries = entries.filter(e => !/\s/.test(e))
+    const multiWordEntries = entries.filter(e => /\s/.test(e))
+
+    // generate a filter containing single-word synonyms
     settings.analysis.filter[`synonyms/${name}`] = {
       "type": "synonym",
-      "synonyms": !_.isEmpty(synonym) ? synonym : ['']
+      "synonyms": !_.isEmpty(singleWordEntries) ? singleWordEntries : ['']
+    };
+
+    // generate a filter containing multi-word synonyms
+    settings.analysis.filter[`synonyms/${name}/multiword`] = {
+      "type": "synonym",
+      "synonyms": !_.isEmpty(multiWordEntries) ? multiWordEntries : ['']
     };
   });
 

diff --git a/synonyms/linter.js b/synonyms/linter.js
@@ -41,6 +41,7 @@ function linter(synonyms) {
       letterCasing(line, logprefix, tokens);
       tokensSanityCheck(line, logprefix, tokens);
       multiWordCheck(line, logprefix, tokens);
+      tokenReplacementCheck(line, logprefix);
       // tokenLengthCheck(line, logprefix, tokens);
     })
   })
@@ -74,6 +75,12 @@ function multiWordCheck(line, logprefix, tokens) {
   });
 }
 
+function tokenReplacementCheck(line, logprefix) {
+  if (/=>/.test(line)) {
+    logger.warn(`${logprefix} synonym rule '=>' is not supported, use ',' instead`);
+  }
+}
+
 function tokenLengthCheck(line, logprefix, tokens) {
   _.each(tokens, token => {
     if (token.length <= 1) {

diff --git a/test/fixtures/expected.json b/test/fixtures/expected.json
@@ -30,6 +30,7 @@
           "filter": [
             "lowercase",
             "trim",
+            "synonyms/custom_admin/multiword",
             "admin_synonyms_multiplexer",
             "icu_folding",
             "word_delimiter",
@@ -48,6 +49,7 @@
           "filter": [
             "lowercase",
             "trim",
+            "synonyms/custom_name/multiword",
             "name_synonyms_multiplexer",
             "icu_folding",
             "remove_ordinals",
@@ -86,6 +88,7 @@
             "lowercase",
             "trim",
             "remove_duplicate_spaces",
+            "synonyms/custom_name/multiword",
             "name_synonyms_multiplexer",
             "icu_folding",
             "remove_ordinals",
@@ -142,6 +145,7 @@
             "lowercase",
             "trim",
             "remove_duplicate_spaces",
+            "synonyms/custom_street/multiword",
             "street_synonyms_multiplexer",
             "icu_folding",
             "remove_ordinals",
@@ -218,18 +222,36 @@
             ""
           ]
         },
+        "synonyms/custom_admin/multiword": {
+          "type": "synonym",
+          "synonyms": [
+            ""
+          ]
+        },
         "synonyms/custom_name": {
           "type": "synonym",
           "synonyms": [
             ""
           ]
         },
+        "synonyms/custom_name/multiword": {
+          "type": "synonym",
+          "synonyms": [
+            ""
+          ]
+        },
         "synonyms/custom_street": {
           "type": "synonym",
           "synonyms": [
             ""
           ]
         },
+        "synonyms/custom_street/multiword": {
+          "type": "synonym",
+          "synonyms": [
+            ""
+          ]
+        },
         "synonyms/directionals": {
           "type": "synonym",
           "synonyms": [
@@ -304,6 +326,12 @@
             "sud,s"
           ]
         },
+        "synonyms/directionals/multiword": {
+          "type": "synonym",
+          "synonyms": [
+            ""
+          ]
+        },
         "synonyms/personal_titles": {
           "type": "synonym",
           "synonyms": [
@@ -500,6 +528,12 @@
             "veuve,vve"
           ]
         },
+        "synonyms/personal_titles/multiword": {
+          "type": "synonym",
+          "synonyms": [
+            ""
+          ]
+        },
         "synonyms/place_names": {
           "type": "synonym",
           "synonyms": [
@@ -819,13 +853,25 @@
             "étang,etang"
           ]
         },
+        "synonyms/place_names/multiword": {
+          "type": "synonym",
+          "synonyms": [
+            ""
+          ]
+        },
         "synonyms/punctuation": {
           "type": "synonym",
           "synonyms": [
             "&,and",
             "&,und"
           ]
         },
+        "synonyms/punctuation/multiword": {
+          "type": "synonym",
+          "synonyms": [
+            ""
+          ]
+        },
         "synonyms/streets": {
           "type": "synonym",
           "synonyms": [
@@ -1639,6 +1685,12 @@
             "wl,well",
             "wls,wells"
           ]
+        },
+        "synonyms/streets/multiword": {
+          "type": "synonym",
+          "synonyms": [
+            ""
+          ]
         }
       },
       "char_filter": {

diff --git a/test/settings.js b/test/settings.js
@@ -83,6 +83,7 @@ module.exports.tests.peliasAdminAnalyzer = function(test, common) {
     t.deepEqual(analyzer.filter, [
       "lowercase",
       "trim",
+      "synonyms/custom_admin/multiword",
       "admin_synonyms_multiplexer",
       "icu_folding",
       "word_delimiter",
@@ -130,6 +131,7 @@ module.exports.tests.peliasIndexOneEdgeGramAnalyzer = function(test, common) {
     t.deepEqual( analyzer.filter, [
       "lowercase",
       "trim",
+      "synonyms/custom_name/multiword",
       "name_synonyms_multiplexer",
       "icu_folding",
       "remove_ordinals",
@@ -186,6 +188,7 @@ module.exports.tests.peliasPhraseAnalyzer = function(test, common) {
       "lowercase",
       "trim",
       "remove_duplicate_spaces",
+      "synonyms/custom_name/multiword",
       "name_synonyms_multiplexer",
       "icu_folding",
       "remove_ordinals",
@@ -293,6 +296,7 @@ module.exports.tests.peliasStreetAnalyzer = function(test, common) {
       "lowercase",
       "trim",
       "remove_duplicate_spaces",
+      "synonyms/custom_street/multiword",
       "street_synonyms_multiplexer",
       "icu_folding",
       "remove_ordinals",