Add ascii string conversion preprocessor

chatterbot/preprocessors.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 """
 Statement pre-processors.
 """
@@ -23,7 +24,7 @@ def clean_whitespace(chatbot, statement):
 def unescape_html(chatbot, statement):
     """
     Convert escaped html characters into unescaped html characters.
-    For example: &lt;b&gt; becomes <b>
+    For example: "&lt;b&gt;" becomes "<b>".
     """
     import sys
 
@@ -37,3 +38,22 @@ def unescape_html(chatbot, statement):
     statement.text = html.unescape(statement.text)
 
     return statement
+
+
+def convert_to_ascii(chatbot, statement):
+    """
+    Converts unicode characters to ASCII character equivalents.
+    For example: "på fédéral" becomes "pa federal".
+    """
+    import unicodedata
+    import sys
+
+    # Normalize unicode characters
+    if sys.version_info[0] < 3:
+        statement.text = unicode(statement.text)
+
+    text = unicodedata.normalize('NFKD', statement.text)
+    text = text.encode('ascii', 'ignore').decode('utf-8')
+
+    statement.text = str(text)
+    return statement

chatterbot/utils.py

@@ -3,22 +3,6 @@
 """
 
 
-def clean(text):
-    """
-    A function for cleaning a string of text.
-    Returns valid ASCII characters.
-    """
-    import unicodedata
-    import sys
-
-    # Normalize unicode characters
-    if sys.version_info[0] < 3:
-        text = unicode(text)
-    text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8')
-
-    return str(text)
-
-
 def import_module(dotted_path):
     """
     Imports the specified module based on the

docs/preprocessors.rst

@@ -26,6 +26,8 @@ ChatterBot comes with several preprocessors build in.
 
 .. autofunction:: chatterbot.preprocessors.unescape_html
 
+.. autofunction:: chatterbot.preprocessors.convert_to_ascii
+
 
 Creating new preprocessors
 ==========================

docs/utils.rst

@@ -10,14 +10,6 @@ Module Imports
 
 .. autofunction:: chatterbot.utils.import_module
 
-String cleaning
----------------
-
-This package of utility contains methods that are usefull
-for cleaning and normalizing strings of text.
-
-.. autofunction:: chatterbot.utils.clean
-
 Terminal input
 --------------
 

tests/test_preprocessors.py

@@ -64,3 +64,16 @@ def test_html_unescape(self):
         cleaned = preprocessors.unescape_html(self.chatbot, statement)
 
         self.assertEqual(cleaned.text, normal_text)
+
+
+class ConvertToASCIIPreprocessorTestCase(ChatBotTestCase):
+    """
+    Make sure that ChatterBot's ASCII conversion preprocessor works as expected.
+    """
+
+    def test_convert_to_ascii(self):
+        statement = Statement(u'Klüft skräms inför på fédéral électoral große')
+        cleaned = preprocessors.convert_to_ascii(self.chatbot, statement)
+        normal_text = 'Kluft skrams infor pa federal electoral groe'
+
+        self.assertEqual(cleaned.text, normal_text)

tests/test_utils.py

@@ -25,14 +25,3 @@ def test_remove_stop_words(self):
         self.assertEqual(len(words), 2)
         self.assertIn('test', list(words))
         self.assertIn('string', list(words))
-
-
-class CleanTests(TestCase):
-
-    def test_non_ascii_chars_replaced(self):
-
-        text = u"Klüft skräms inför på fédéral électoral große"
-        clean_text = utils.clean(text)
-        normal_text = "Kluft skrams infor pa federal electoral groe"
-
-        self.assertEqual(clean_text, normal_text)