access simplemma functionality only via annif.simplemma_util

annif/analyzer/simplemma.py

@@ -1,9 +1,7 @@
 """Simplemma analyzer for Annif, based on simplemma lemmatizer."""
 from __future__ import annotations
 
-import simplemma
-
-import annif.langsupport
+import annif.simplemma_util
 
 from . import analyzer
 
@@ -13,10 +11,7 @@ class SimplemmaAnalyzer(analyzer.Analyzer):
 
     def __init__(self, param: str, **kwargs) -> None:
         self.lang = param
-        self.lemmatizer = simplemma.Lemmatizer(
-            lemmatization_strategy=annif.langsupport.lemmatization_strategy
-        )
         super().__init__(**kwargs)
 
     def _normalize_word(self, word: str) -> str:
-        return self.lemmatizer.lemmatize(word, lang=self.lang)
+        return annif.simplemma_util.lemmatizer.lemmatize(word, lang=self.lang)

annif/simplemma_util.py

@@ -0,0 +1,15 @@
+"""Wrapper code for using Simplemma functionality in Annif"""
+
+from simplemma import LanguageDetector, Lemmatizer
+from simplemma.strategies import DefaultStrategy
+from simplemma.strategies.dictionaries import DefaultDictionaryFactory
+
+LANG_CACHE_SIZE = 5  # How many language dictionaries to keep in memory at once (max)
+
+_dictionary_factory = DefaultDictionaryFactory(cache_max_size=LANG_CACHE_SIZE)
+_lemmatization_strategy = DefaultStrategy(dictionary_factory=_dictionary_factory)
+lemmatizer = Lemmatizer(lemmatization_strategy=_lemmatization_strategy)
+
+
+def get_language_detector(lang: str) -> LanguageDetector:
+    return LanguageDetector(lang, lemmatization_strategy=_lemmatization_strategy)

annif/transform/langfilter.py

@@ -4,10 +4,8 @@
 
 from typing import TYPE_CHECKING
 
-from simplemma import LanguageDetector
-
 import annif
-import annif.langsupport
+import annif.simplemma_util
 
 from . import transform
 
@@ -31,9 +29,8 @@ def __init__(
         self.text_min_length = int(text_min_length)
         self.sentence_min_length = int(sentence_min_length)
         self.min_ratio = float(min_ratio)
-        self.language_detector = LanguageDetector(
-            self.project.language,
-            lemmatization_strategy=annif.langsupport.lemmatization_strategy,
+        self.language_detector = annif.simplemma_util.get_language_detector(
+            self.project.language
         )
 
     def transform_fn(self, text: str) -> str: