Switch from SimepleStemmer to PosStemmer

chatterbot/stemming.py

@@ -207,7 +207,7 @@ def get_bigram_pair_string(self, text):
 
         for index in range(1, word_count):
             word = words[index].lower()
-            previous_word_pos = pos_tags[index - 1][1]
+            previous_word_pos = pos_tags[index][1]
             if word not in self.get_stopwords() and len(word) > 1:
                 bigram = previous_word_pos + ':' + hypernyms[index].lower()
                 high_quality_bigrams.append(bigram)

chatterbot/storage/storage_adapter.py

@@ -1,5 +1,5 @@
 import logging
-from chatterbot.stemming import SimpleStemmer
+from chatterbot.stemming import PosHypernymStemmer
 
 
 class StorageAdapter(object):
@@ -16,7 +16,7 @@ def __init__(self, *args, **kwargs):
         self.logger = kwargs.get('logger', logging.getLogger(__name__))
         self.adapter_supports_queries = True
 
-        self.stemmer = SimpleStemmer(language=kwargs.get(
+        self.stemmer = PosHypernymStemmer(language=kwargs.get(
             'stemmer_language', 'english'
         ))
 

chatterbot/trainers.py

@@ -5,7 +5,7 @@
 from multiprocessing import Pool, Manager
 from dateutil import parser as date_parser
 from chatterbot.conversation import Statement
-from chatterbot.stemming import SimpleStemmer
+from chatterbot.stemming import PosHypernymStemmer
 from chatterbot import utils
 
 
@@ -30,7 +30,7 @@ def __init__(self, chatbot, **kwargs):
             environment_default
         )
 
-        self.stemmer = SimpleStemmer(language=kwargs.get(
+        self.stemmer = PosHypernymStemmer(language=kwargs.get(
             'stemmer_language', 'english'
         ))
 
@@ -430,7 +430,7 @@ def track_progress(members):
     def train(self):
         import glob
 
-        stemmer = SimpleStemmer(language=self.stemmer.language)
+        stemmer = PosHypernymStemmer(language=self.stemmer.language)
 
         # Download and extract the Ubuntu dialog corpus if needed
         corpus_download_path = self.download(self.data_download_url)

tests/logic/best_match_integration_tests/test_levenshtein_distance.py

@@ -49,7 +49,10 @@ def test_confidence_exact_match(self):
         self.assertEqual(match.confidence, 1)
 
     def test_confidence_half_match(self):
-        self.chatbot.storage.create(text='xxyy', in_response_to='xxyy')
+        # Assume that the storage adapter returns a partial match
+        self.adapter.chatbot.storage.filter = MagicMock(return_value=[
+            Statement(text='xxyy')
+        ])
 
         statement = Statement(text='wwxx')
         match = self.adapter.get(statement)

tests/test_chatbot.py

@@ -12,7 +12,8 @@ def test_get_initialization_functions(self):
         functions = self.chatbot.get_initialization_functions()
 
         self.assertIn('initialize_nltk_stopwords', functions)
-        self.assertIsLength(functions, 1)
+        self.assertIn('initialize_nltk_wordnet', functions)
+        self.assertIsLength(functions, 2)
 
     def test_get_initialization_functions_synset_distance(self):
         """
@@ -38,8 +39,9 @@ def test_get_initialization_functions_sentiment_comparison(self):
         functions = self.chatbot.get_initialization_functions()
 
         self.assertIn('initialize_nltk_stopwords', functions)
+        self.assertIn('initialize_nltk_wordnet', functions)
         self.assertIn('initialize_nltk_vader_lexicon', functions)
-        self.assertIsLength(functions, 2)
+        self.assertIsLength(functions, 3)
 
     def test_get_initialization_functions_jaccard_similarity(self):
         """

tests/test_stemming.py

@@ -131,7 +131,7 @@ def test_stemming(self):
             'Hello, how are you doing on this awesome day?'
         )
 
-        self.assertEqual(stemmed_text, 'DT:awesome JJ:time_unit')
+        self.assertEqual(stemmed_text, 'JJ:awesome NN:time_unit')
 
     def test_string_becomes_lowercase(self):
         stemmed_text = self.stemmer.get_bigram_pair_string('THIS IS HOW IT BEGINS!')
@@ -141,12 +141,12 @@ def test_string_becomes_lowercase(self):
     def test_stemming_medium_sized_words(self):
         stemmed_text = self.stemmer.get_bigram_pair_string('Hello, my name is Gunther.')
 
-        self.assertEqual(stemmed_text, 'PRP$:language_unit VBZ:gunther')
+        self.assertEqual(stemmed_text, 'NN:language_unit NNP:gunther')
 
     def test_stemming_long_words(self):
         stemmed_text = self.stemmer.get_bigram_pair_string('I play several orchestra instruments for pleasuer.')
 
-        self.assertEqual(stemmed_text, 'PRP:compete VBP:several JJ:orchestra JJ:device IN:pleasuer')
+        self.assertEqual(stemmed_text, 'VBP:compete JJ:several JJ:orchestra NNS:device NN:pleasuer')
 
     def test_get_bigram_pair_string_punctuation_only(self):
         bigram_string = self.stemmer.get_bigram_pair_string(
@@ -209,18 +209,18 @@ def test_get_bigram_pair_string_multiple_words(self):
             'Hello Dr. Salazar. How are you today?'
         )
 
-        self.assertEqual(bigram_string, 'NNP:scholar NNP:salazar. PRP:present')
+        self.assertEqual(bigram_string, 'NNP:scholar NNP:salazar. NN:present')
 
     def test_get_bigram_pair_string_single_character_words(self):
         bigram_string = self.stemmer.get_bigram_pair_string(
             'a e i o u'
         )
 
-        self.assertEqual(bigram_string, 'DT:e NN:i NN:o VBP:u')
+        self.assertEqual(bigram_string, 'NN:e NN:i VBP:o NN:u')
 
     def test_get_bigram_pair_string_two_character_words(self):
         bigram_string = self.stemmer.get_bigram_pair_string(
             'Lo my mu it is of us'
         )
 
-        self.assertEqual(bigram_string, 'PRP$:letter IN:us')
+        self.assertEqual(bigram_string, 'NN:letter PRP:us')

tests/training/test_chatterbot_corpus_training.py

@@ -29,15 +29,15 @@ def test_train_with_english_greeting_corpus_search_text(self):
         results = list(self.chatbot.storage.filter(text='Hello'))
 
         self.assertGreater(len(results), 1)
-        self.assertEqual(results[0].search_text, 'ell')
+        self.assertEqual(results[0].search_text, 'hello')
 
     def test_train_with_english_greeting_corpus_search_in_response_to(self):
         self.trainer.train('chatterbot.corpus.english.greetings')
 
         results = list(self.chatbot.storage.filter(in_response_to='Hello'))
 
         self.assertGreater(len(results), 1)
-        self.assertEqual(results[0].search_in_response_to, 'ell')
+        self.assertEqual(results[0].search_in_response_to, 'hello')
 
     def test_train_with_english_greeting_corpus_tags(self):
         self.trainer.train('chatterbot.corpus.english.greetings')

tests/training/test_list_training.py

@@ -65,7 +65,7 @@ def test_training_sets_search_text(self):
         ))
 
         self.assertIsLength(statements, 1)
-        self.assertEqual(statements[0].search_text, "ik")
+        self.assertEqual(statements[0].search_text, 'IN:kind NN:headdress')
 
     def test_training_sets_search_in_response_to(self):
 
@@ -81,7 +81,7 @@ def test_training_sets_search_in_response_to(self):
         ))
 
         self.assertIsLength(statements, 1)
-        self.assertEqual(statements[0].search_in_response_to, "ik")
+        self.assertEqual(statements[0].search_in_response_to, 'IN:kind NN:headdress')
 
     def test_database_has_correct_format(self):
         """

tests/training/test_twitter_trainer.py

@@ -88,11 +88,11 @@ def test_train_sets_search_text(self):
         statements = list(self.trainer.chatbot.storage.filter())
 
         self.assertGreater(len(statements), 1)
-        self.assertEqual(statements[0].search_text, 'urub')
+        self.assertEqual(statements[0].search_text, 'VBP:sure NNP:jewel')
 
     def test_train_sets_search_in_response_to(self):
         self.trainer.train()
         statements = list(self.trainer.chatbot.storage.filter())
 
         self.assertGreater(len(statements), 1)
-        self.assertEqual(statements[0].search_in_response_to, 'urub')
+        self.assertEqual(statements[0].search_in_response_to, 'VBP:sure NNP:jewel')

tests/training/test_ubuntu_corpus_training.py

@@ -176,7 +176,7 @@ def test_train_sets_search_text(self):
         results = list(self.chatbot.storage.filter(text='Is anyone there?'))
 
         self.assertEqual(len(results), 2)
-        self.assertEqual(results[0].search_text, 'nyon')
+        self.assertEqual(results[0].search_text, 'NN:anyone')
 
     def test_train_sets_search_in_response_to(self):
         """
@@ -190,7 +190,7 @@ def test_train_sets_search_in_response_to(self):
         results = list(self.chatbot.storage.filter(in_response_to='Is anyone there?'))
 
         self.assertEqual(len(results), 2)
-        self.assertEqual(results[0].search_in_response_to, 'nyon')
+        self.assertEqual(results[0].search_in_response_to, 'NN:anyone')
 
     def test_is_extracted(self):
         """