Add filter option for search_text_contains

chatterbot/storage/django_storage.py

@@ -44,6 +44,7 @@ def filter(self, **kwargs):
         exclude_text = kwargs.pop('exclude_text', None)
         exclude_text_words = kwargs.pop('exclude_text_words', [])
         persona_not_startswith = kwargs.pop('persona_not_startswith', None)
+        search_text_contains = kwargs.pop('search_text_contains', None)
 
         # Convert a single sting into a list if only one tag is provided
         if type(tags) == str:
@@ -73,6 +74,16 @@ def filter(self, **kwargs):
                 persona__startswith='bot:'
             )
 
+        if search_text_contains:
+            or_query = Q()
+
+            for word in search_text_contains.split(' '):
+                or_query |= Q(search_text__contains=word)
+
+            statements = statements.filter(
+                or_query
+            )
+
         if order_by:
             statements = statements.order_by(*order_by)
 

chatterbot/storage/mongodb.py

@@ -80,6 +80,7 @@ def filter(self, **kwargs):
         exclude_text = kwargs.pop('exclude_text', None)
         exclude_text_words = kwargs.pop('exclude_text_words', [])
         persona_not_startswith = kwargs.pop('persona_not_startswith', None)
+        search_text_contains = kwargs.pop('search_text_contains', None)
 
         if tags:
             kwargs['tags'] = {
@@ -119,6 +120,12 @@ def filter(self, **kwargs):
                 }
             kwargs['persona']['$not'] = re.compile('^bot:*')
 
+        if search_text_contains:
+            or_regex = '|'.join([
+                '{}'.format(word) for word in search_text_contains.split(' ')
+            ])
+            kwargs['search_text'] = re.compile(or_regex)
+
         mongo_ordering = []
 
         if order_by:

chatterbot/storage/sql_storage.py

@@ -118,6 +118,7 @@ def filter(self, **kwargs):
         exclude_text = kwargs.pop('exclude_text', None)
         exclude_text_words = kwargs.pop('exclude_text_words', [])
         persona_not_startswith = kwargs.pop('persona_not_startswith', None)
+        search_text_contains = kwargs.pop('search_text_contains', None)
 
         # Convert a single sting into a list if only one tag is provided
         if type(tags) == str:
@@ -151,6 +152,14 @@ def filter(self, **kwargs):
                 ~Statement.persona.startswith('bot:')
             )
 
+        if search_text_contains:
+            or_query = [
+                Statement.search_text.contains(word) for word in search_text_contains.split(' ')
+            ]
+            statements = statements.filter(
+                or_(*or_query)
+            )
+
         if order_by:
 
             if 'created_at' in order_by:

chatterbot/trainers.py

@@ -429,9 +429,8 @@ def track_progress(members):
 
     def train(self):
         import glob
-        from chatterbot.stemming import SimpleStemmer
 
-        stemmer = SimpleStemmer()
+        stemmer = SimpleStemmer(language=self.stemmer.language)
 
         # Download and extract the Ubuntu dialog corpus if needed
         corpus_download_path = self.download(self.data_download_url)

tests/storage/test_mongo_adapter.py

@@ -273,6 +273,27 @@ def test_persona_not_startswith(self):
         self.assertEqual(len(results), 1)
         self.assertEqual(results[0].text, 'Hi everyone!')
 
+    def test_search_text_contains(self):
+        self.adapter.create(text='Hello!', search_text='hello exclamation')
+        self.adapter.create(text='Hi everyone!', search_text='hi everyone')
+
+        results = list(self.adapter.filter(
+            search_text_contains='everyone'
+        ))
+
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0].text, 'Hi everyone!')
+
+    def test_search_text_contains_multiple_matches(self):
+        self.adapter.create(text='Hello!', search_text='hello exclamation')
+        self.adapter.create(text='Hi everyone!', search_text='hi everyone')
+
+        results = list(self.adapter.filter(
+            search_text_contains='hello everyone'
+        ))
+
+        self.assertEqual(len(results), 2)
+
 
 class MongoOrderingTestCase(MongoAdapterTestCase):
     """

tests/storage/test_sql_adapter.py

@@ -224,6 +224,27 @@ def test_persona_not_startswith(self):
         self.assertEqual(len(results), 1)
         self.assertEqual(results[0].text, 'Hi everyone!')
 
+    def test_search_text_contains(self):
+        self.adapter.create(text='Hello!', search_text='hello exclamation')
+        self.adapter.create(text='Hi everyone!', search_text='hi everyone')
+
+        results = list(self.adapter.filter(
+            search_text_contains='everyone'
+        ))
+
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0].text, 'Hi everyone!')
+
+    def test_search_text_contains_multiple_matches(self):
+        self.adapter.create(text='Hello!', search_text='hello exclamation')
+        self.adapter.create(text='Hi everyone!', search_text='hi everyone')
+
+        results = list(self.adapter.filter(
+            search_text_contains='hello everyone'
+        ))
+
+        self.assertEqual(len(results), 2)
+
 
 class SQLOrderingTests(SQLStorageAdapterTestCase):
     """

tests_django/test_django_adapter.py

@@ -277,6 +277,27 @@ def test_persona_not_startswith(self):
         self.assertEqual(len(results), 1)
         self.assertEqual(results[0].text, 'Hi everyone!')
 
+    def test_search_text_contains(self):
+        self.adapter.create(text='Hello!', search_text='hello exclamation')
+        self.adapter.create(text='Hi everyone!', search_text='hi everyone')
+
+        results = list(self.adapter.filter(
+            search_text_contains='everyone'
+        ))
+
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0].text, 'Hi everyone!')
+
+    def test_search_text_contains_multiple_matches(self):
+        self.adapter.create(text='Hello!', search_text='hello exclamation')
+        self.adapter.create(text='Hi everyone!', search_text='hi everyone')
+
+        results = list(self.adapter.filter(
+            search_text_contains='hello everyone'
+        ))
+
+        self.assertEqual(len(results), 2)
+
 
 class DjangoOrderingTests(DjangoAdapterTestCase):
     """