⚡ code optimization for cleaning special chars from string

deepchecks/nlp/utils/text_embeddings.py

@@ -9,6 +9,7 @@
 # ----------------------------------------------------------------------------
 #
 """Utils module for calculating embeddings for text."""
+import re
 import sys
 import warnings
 from itertools import islice
@@ -22,6 +23,10 @@
 EMBEDDING_CTX_LENGTH = 8191
 EMBEDDING_ENCODING = 'cl100k_base'
 
+PATTERN_SPECIAL_CHARS = re.compile(r"[!@#$%^&*()_+{}|:\"<>?~`\-=\[\]\;',.\/]")
+PATTERN_SPACE_CHARS = re.compile(r'\s')
+PATTERN_BR_CHARS = re.compile(r'<br />')
+
 
 def batched(iterable, n):
     """Batch data into tuples of length n. The last batch may be shorter."""
@@ -205,10 +210,21 @@ def len_safe_get_embedding(list_of_texts, model_name=EMBEDDING_MODEL, max_tokens
     return embeddings
 
 
-def _clean_special_chars(text):
-    special_chars = r'!@#$%^&*()_+{}|:"<>?~`-=[]\;\',./'
-    for char in special_chars:
-        text = text.replace(char, '')
-    text = text.replace('\n', ' ').replace('\r', ' ').replace('\t', ' ')
-    text = text.replace('<br />', ' ')
+def _clean_special_chars(text: str) -> str:
+    """
+    Remove special characters, replaces space characters with space.
+
+    Parameters
+    ----------
+    text : str
+        The `text` parameter is a string that represents the input text that needs to be cleaned.
+
+    Returns
+    -------
+        text
+            Cleaned text string
+    """
+    text = PATTERN_SPECIAL_CHARS.sub('', text)
+    text = PATTERN_SPACE_CHARS.sub(' ', text)
+    text = PATTERN_BR_CHARS.sub(' ', text)
     return text