Fix search index diacritics handling

Closes #210: ES asciifolding filter doesn't correctly handle multiple diacritics
VedaWebProject · May 2, 2024 · 3a1150f · 3a1150f
1 parent 8998f31
commit 3a1150f
Show file tree

Hide file tree

Showing 13 changed files with 44 additions and 42 deletions.
diff --git a/Tekst-API/Dockerfile b/Tekst-API/Dockerfile
@@ -104,7 +104,7 @@ RUN groupadd -g 1337 tekst && \
     useradd -m -u 1337 -g tekst tekst
 
 # copy WSGI config
-COPY ./deploy/gunicorn_conf.py ./
+COPY ./deploy/gunicorn/gunicorn_conf.py ./
 COPY ./deploy/entrypoint.sh /
 
 USER tekst

diff --git a/Tekst-API/deploy/elasticsearch/Dockerfile b/Tekst-API/deploy/elasticsearch/Dockerfile
@@ -0,0 +1,2 @@
+FROM elasticsearch:8.12.2
+RUN elasticsearch-plugin install analysis-icu
diff --git a/Tekst-API/deploy/gunicorn_conf.py → Tekst-API/deploy/gunicorn/gunicorn_conf.py b/Tekst-API/deploy/gunicorn_conf.py → Tekst-API/deploy/gunicorn/gunicorn_conf.py
diff --git a/Tekst-API/dev/docker-compose.yml b/Tekst-API/dev/docker-compose.yml
@@ -45,7 +45,10 @@ services:
       - dev
 
   es:
-    image: elasticsearch:8.12.2
+    build:
+      context: ../deploy/elasticsearch
+      tags:
+        - "elasticsearch:tekst"
     deploy:
       resources:
         limits:

diff --git a/Tekst-API/tekst/resources/__init__.py b/Tekst-API/tekst/resources/__init__.py
@@ -150,7 +150,7 @@ def index_doc_props(cls) -> dict[str, Any]:
         return dict(
             comment={
                 "type": "text",
-                "analyzer": "standard_asciifolding",
+                "analyzer": "standard_no_diacritics",
                 "fields": {"strict": {"type": "text"}},
             },
             **cls.rtype_index_doc_props(),

diff --git a/Tekst-API/tekst/resources/plain_text.py b/Tekst-API/tekst/resources/plain_text.py
@@ -42,7 +42,7 @@ def rtype_index_doc_props(cls) -> dict[str, Any]:
         return {
             "text": {
                 "type": "text",
-                "analyzer": "standard_asciifolding",
+                "analyzer": "standard_no_diacritics",
                 "fields": {"strict": {"type": "text"}},
             },
         }

diff --git a/Tekst-API/tekst/resources/rich_text.py b/Tekst-API/tekst/resources/rich_text.py
@@ -35,7 +35,7 @@ def rtype_index_doc_props(cls) -> dict[str, Any]:
         return {
             "html": {
                 "type": "text",
-                "analyzer": "standard_asciifolding",
+                "analyzer": "standard_no_diacritics",
                 "fields": {"strict": {"type": "text"}},
             },
         }

diff --git a/Tekst-API/tekst/resources/text_annotation.py b/Tekst-API/tekst/resources/text_annotation.py
@@ -203,7 +203,7 @@ def rtype_index_doc_props(cls) -> dict[str, Any]:
                 "properties": {
                     "token": {
                         "type": "keyword",
-                        "normalizer": "asciifolding_normalizer",
+                        "normalizer": "no_diacritics_normalizer",
                         "fields": {"strict": {"type": "keyword"}},
                     }
                 },

diff --git a/Tekst-API/tekst/search/templates.py b/Tekst-API/tekst/search/templates.py
@@ -14,6 +14,29 @@
 
 IDX_TEMPLATE = {
     "aliases": {IDX_ALIAS: {}},
+    "settings": {
+        "index": {"number_of_shards": 1, "number_of_replicas": 0},
+        "analysis": {
+            "analyzer": {
+                "standard_no_diacritics": {
+                    "tokenizer": "standard",
+                    "filter": ["no_diacritics", "lowercase"],
+                },
+            },
+            "filter": {
+                "no_diacritics": {
+                    "type": "icu_transform",
+                    "id": "NFD; [:Nonspacing Mark:] Remove; NFC",
+                }
+            },
+            "normalizer": {
+                "no_diacritics_normalizer": {
+                    "type": "custom",
+                    "filter": ["no_diacritics", "lowercase"],
+                },
+            },
+        },
+    },
     "mappings": {
         "dynamic": False,
         "properties": {
@@ -28,42 +51,13 @@
                     "path_match": "*.annotations.*",
                     "mapping": {
                         "type": "keyword",
-                        "normalizer": "asciifolding_normalizer",
+                        "normalizer": "no_diacritics_normalizer",
                         "fields": {"strict": {"type": "keyword"}},
                     },
                 }
             }
         ],
     },
-    "settings": {
-        "index": {"number_of_shards": 1, "number_of_replicas": 0},
-        "analysis": {
-            "analyzer": {
-                "standard_asciifolding": {
-                    "tokenizer": "standard",
-                    "filter": ["asciifolding", "lowercase"],
-                },
-            },
-            "filter": {
-                "asciifolding_preserve": {
-                    "type": "asciifolding",
-                    "preserve_original": True,
-                }
-            },
-            "normalizer": {
-                "asciifolding_normalizer": {
-                    "type": "custom",
-                    "char_filter": [],
-                    "filter": ["asciifolding", "lowercase"],
-                },
-                "asciifolding_normalizer_preserve_case": {
-                    "type": "custom",
-                    "char_filter": [],
-                    "filter": ["asciifolding"],
-                },
-            },
-        },
-    },
 }
 
 _GENERAL_SOURCE_INCLUDES = [

diff --git a/Tekst-API/tekst/utils/strings.py b/Tekst-API/tekst/utils/strings.py
@@ -19,7 +19,7 @@ def safe_name(
     if isinstance(string, bytes):
         string = string.decode()
 
-    string = remove_diacritics(string)
+    string = no_diacritics(string)
 
     # lowercase and delimit using underscores
     string = re.sub(r"[^a-z0-9]+", delim, string.lower()).strip(delim)
@@ -33,7 +33,7 @@ def safe_name(
     return string
 
 
-def remove_diacritics(string: str) -> str:
+def no_diacritics(string: str) -> str:
     """Removes diacritics from the input string and returns it NFC-normalized"""
     return "".join(
         ucdata.normalize("NFC", c)

diff --git a/Tekst-API/tests/test_utils_strings.py b/Tekst-API/tests/test_utils_strings.py
@@ -21,5 +21,5 @@ def test_safe_name():
     assert strings.safe_name("foo bar", delim="+") == "foo+bar"
 
 
-def test_remove_diacritics():
-    assert strings.remove_diacritics("Hörglwàartŝ") == "Horglwaarts"
+def test_no_diacritics():
+    assert strings.no_diacritics("Hörglwàartŝ") == "Horglwaarts"
diff --git a/Tekst-Web/src/views/ResourcesView.vue b/Tekst-Web/src/views/ResourcesView.vue
@@ -303,7 +303,7 @@ async function handleImportClick(resource: AnyResourceRead) {
     });
     if (!error) {
       addTask(data);
-      message.info($t('contents.msgImportInfo'), undefined, 20);
+      message.info($t('contents.msgImportInfo'), undefined, 5);
       startTasksPolling();
     }
     actionsLoading.value = false;

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -68,7 +68,10 @@ services:
     command: mongod --setParameter diagnosticDataCollectionEnabled=false
 
   es:
-    image: elasticsearch:8.12.2
+    build:
+      context: ./Tekst-API/deploy/elasticsearch
+      tags:
+        - "elasticsearch:tekst"
     deploy:
       resources:
         limits: