Updated unit tests following sql storage engine

src/programy/bot.py

@@ -20,10 +20,10 @@
 
 from programy.brain import Brain
 from programy.dialog.dialog import Conversation, Question, Sentence
-from programy.dialog.storage.factory import ConversationStorageFactory
+from programy.dialog.conversation import ConversationManager
 from programy.config.bot.bot import BotConfiguration
 from programy.utils.classes.loader import ClassLoader
-
+from programy.spelling.base import SpellingChecker
 
 class BrainSelector(object):
 
@@ -94,9 +94,8 @@ def __init__(self, config, client):
         self._spell_checker = None
         self.initiate_spellchecker()
 
-        self._conversations = {}
-        self._conversation_storage = None
-        self.initiate_conversation_storage()
+        self._conversation_mgr = ConversationManager(config.conversations)
+        self._conversation_mgr.initialise(self._client.storage_factory)
 
     def ylogger_type(self):
         return "bot"
@@ -118,18 +117,9 @@ def brain_factory(self):
         return self._brain_factory
 
     def initiate_spellchecker(self):
-        # TODO Move this to Spelling bass class
         if self.configuration is not None:
-            if self.configuration.spelling.classname is not None:
-                try:
-                    YLogger.info(self, "Loading spelling checker from class [%s]", self.configuration.spelling.classname)
-                    spell_class = ClassLoader.instantiate_class(self.configuration.spelling.classname)
-                    self._spell_checker = spell_class(self.configuration.spelling)
-                    self._spell_checker.initialise(self.client)
-                except Exception as excep:
-                    YLogger.exception(self, "Failed to initiate spellcheker", excep)
-            else:
-                YLogger.warning(self, "No configuration setting for spelling checker!")
+            if self.configuration.spelling is not None:
+                self._spell_checker = SpellingChecker.initiate_spellchecker(self.configuration.spelling, self.client.storage_factory)
 
     @property
     def spell_checker(self):
@@ -141,7 +131,7 @@ def brain(self):
 
     @property
     def conversations(self):
-        return self._conversations
+        return self._conversation_mgr
 
     @property
     def default_response(self):
@@ -202,77 +192,24 @@ def get_version_string(self, client_context):
                 client_context.brain.properties.property("birthdate"))
 
     def has_conversation(self, client_context):
-        return bool(client_context.userid in self._conversations)
+        return self._conversation_mgr.has_conversation(client_context)
 
     def conversation(self, client_context):
         return self.get_conversation(client_context)
 
     def get_conversation(self, client_context):
-        # TODO MOve into Conversation Manager
-        if client_context.userid in self._conversations:
-            YLogger.info(client_context, "Retrieving conversation for client %s", client_context.userid)
-            return self._conversations[client_context.userid]
-
-        else:
-            YLogger.info(client_context, "Creating new conversation for client %s", client_context.userid)
-
-            conversation = Conversation(client_context)
+        return self._conversation_mgr.get_conversation(client_context)
 
-            if client_context.brain.properties is not None:
-                conversation.load_initial_variables(client_context.brain.variables)
+    def save_conversation(self, client_context):
+        self._conversation_mgr.save_conversation(client_context)
 
-            self._conversations[client_context.userid] = conversation
-
-            self.load_conversation(client_context.userid)
-
-            return conversation
-
-    def initiate_conversation_storage(self):
-        # TODO MOve into Conversation Manager
-        pass
-        """
-        if self.configuration is not None:
-            if self.configuration.conversations is not None:
-                self._conversation_storage = ConversationStorageFactory.get_storage(self.configuration)
-                if self._conversation_storage is not None:
-                    if self.configuration.conversations.empty_on_start is True:
-                        self._conversation_storage.empty ()
-        """
-
-    def load_conversation(self, clientid):
-        # TODO MOve into Conversation Manager
-        if self._conversation_storage is not None:
-            if clientid in self._conversations:
-                conversation = self._conversations[clientid]
-                self._conversation_storage.load_conversation(conversation, clientid,
-                                                             self.configuration.conversations.restore_last_topic)
-
-    def save_conversation(self, clientid):
-        # TODO MOve into Conversation Manager
-        if self._conversation_storage is not None:
-            if clientid in self._conversations:
-                conversation = self._conversations[clientid]
-                self._conversation_storage.save_conversation(conversation, clientid)
-            else:
-                YLogger.error(self, "Unknown conversation id type [%s] unable tonot persist!", clientid)
-
-    def check_spelling_before(self, each_sentence):
-        # TODO Move this to spelliing base class
-        if self.configuration.spelling.check_before is True:
-            text = each_sentence.text()
-            corrected = self.spell_checker.correct(text)
-            YLogger.debug(self, "Spell Checker corrected [%s] to [%s]", text, corrected)
-            each_sentence.replace_words(corrected)
+    def check_spelling_before(self, client_context, each_sentence):
+        if self.spell_checker is not None:
+            self.spell_checker.check_spelling_before(client_context, each_sentence)
 
     def check_spelling_and_retry(self, client_context, each_sentence):
-        # TODO Move this to spelling base class
-        if self.configuration.spelling.check_and_retry is True:
-            text = each_sentence.text()
-            corrected = self.spell_checker.correct(text)
-            YLogger.debug(self, "Spell Checker corrected [%s] to [%s]", text, corrected)
-            each_sentence.replace_words(corrected)
-            response = client_context.brain.ask_question(client_context, each_sentence)
-            return response
+        if self.spell_checker is not None:
+            return self.spell_checker.check_spelling_and_retry(client_context, each_sentence)
         return None
 
     def get_default_response(self, client_context):
@@ -370,26 +307,16 @@ def ask_question(self, client_context, text, srai=False, responselogger=None):
         if srai is True:
             conversation.pop_dialog()
 
-        response = self.combine_answers(answers)
-
-        self.log_question_and_answer(client_context, text, response)
+        self.save_conversation(client_context)
 
-        return response
-
-    def log_question_and_answer(self, client_context, text, response):
-        # TODO Move to Conversation Storage Class
-        convo_logger = logging.getLogger("conversation")
-        if convo_logger:
-            qanda =  "%s - Question[%s], Response[%s]"%(str(client_context), text, response)
-            convo_logger.info(qanda)
+        return self.combine_answers(answers)
 
     def process_sentence(self, client_context, sentence, srai, responselogger):
-        # TODO Create Sentence Processor
         client_context.check_max_recursion()
         client_context.check_max_timeout()
 
         if srai is False:
-            self.check_spelling_before(sentence)
+            self.check_spelling_before(client_context, sentence)
 
         response = client_context.brain.ask_question(client_context, sentence, srai)
 
@@ -401,18 +328,14 @@ def process_sentence(self, client_context, sentence, srai, responselogger):
         else:
             return self.handle_none_response(client_context, sentence, responselogger)
 
-        return answer
-
     def handle_response(self, client_context, sentence, response, srai, responselogger):
-        # TODO Create Response Processor
         YLogger.debug(client_context, "Raw Response (%s): %s", client_context.userid, response)
         sentence.response = response
         answer = self.post_process_response(client_context, response, srai)
         self.log_answer(client_context, sentence.text, answer, responselogger)
         return answer
 
     def handle_none_response(self, clientid, sentence, responselogger):
-        # TODO Create Response Processor
         sentence.response = self.get_default_response(clientid)
         if responselogger is not None:
             responselogger.log_unknown_response(sentence)

src/programy/brain.py

@@ -65,8 +65,7 @@ def __init__(self, bot, configuration: BrainConfiguration):
         self._maps_collection = MapCollection()
 
         self._properties_collection = PropertiesCollection()
-        self._variables_collection = DefaultVariablesCollection()
-        self._defaults_collection = RegexTemplatesCollection()
+        self._default_variables_collection = DefaultVariablesCollection()
 
         self._preprocessors = PreProcessorCollection()
         self._postprocessors = PostProcessorCollection()
@@ -145,8 +144,8 @@ def properties(self):
         return self._properties_collection
 
     @property
-    def variables(self):
-        return self._variables_collection
+    def default_variables(self):
+        return self._default_variables_collection
 
     @property
     def preprocessors(self):
@@ -220,7 +219,7 @@ def load_binary(self, binaries_configuration):
         YLogger.info(self, "Loading binary brain from [%s]", binaries_configuration.binary_filename)
         try:
 
-            storage_engine = self.bot.client.storage_factory.storage_engine(binaries_configuration.storage)
+            storage_engine = self.bot.client.storage_factory.entity_storage_engine(binaries_configuration.storage)
             binaries_storage = storage_engine.binaries_storage()
             self._aiml_parser = binaries_storage.load_binary()
 
@@ -236,10 +235,10 @@ def load_binary(self, binaries_configuration):
     def save_binary(self, binaries_configuration):
         # TODO MOve into BinariesManager
         try:
-            if self.bot.client.storage_factor.storage_engine_available(StorageFactory.BINARIES) is True:
+            if self.bot.client.storage_factor.entity_storage_engine_available(StorageFactory.BINARIES) is True:
                 YLogger.info(self, "Saving binary brain to [%s]", StorageFactory.BINARIES)
 
-                storage_engine = self.bot.client.storage_factory.storage_engine(StorageFactory.BINARIES)
+                storage_engine = self.bot.client.storage_factory.entity_storage_engine(StorageFactory.BINARIES)
                 binaries_storage = storage_engine.binaries_storage()
                 binaries_storage.save_binary(self._aiml_parser)
 
@@ -310,9 +309,9 @@ def _load_properties(self):
         self._properties_collection.empty()
         self._properties_collection.load(self.bot.client.storage_factory)
 
-    def _load_variables(self):
-        self._variables_collection.empty()
-        self._variables_collection.load(self.bot.client.storage_factory)
+    def _load_default_variables(self):
+        self._default_variables_collection.empty()
+        self._default_variables_collection.load(self.bot.client.storage_factory)
 
     def _load_maps(self):
         self._maps_collection.empty()
@@ -361,7 +360,7 @@ def load_collections(self):
         self._load_persons()
         self._load_person2s()
         self._load_properties()
-        self._load_variables()
+        self._load_default_variables()
         self._load_rdfs()
         self._load_sets()
         self._load_maps()
@@ -458,8 +457,8 @@ def process_oob(self, client_context, oob_command):
         return ""
 
     def load_regex_templates(self):
-        if self.bot.client.storage_factory.storage_engine_available(StorageFactory.REGEX_TEMPLATES) is True:
-            storage_engine = self.bot.client.storage_factory.storage_engine(StorageFactory.REGEX_TEMPLATES)
+        if self.bot.client.storage_factory.entity_storage_engine_available(StorageFactory.REGEX_TEMPLATES) is True:
+            storage_engine = self.bot.client.storage_factory.entity_storage_engine(StorageFactory.REGEX_TEMPLATES)
             regex_store = storage_engine.regex_store()
             regex_store.load(self._regex_templates)
 

src/programy/chatbot.py

@@ -11,7 +11,7 @@ def __init__(self, argument_parser=None):
         ConsoleBotClient.__init__(self, argument_parser)
 
     def parse_configuration(self):
-        self.configuration.client_configuration.storage._category_storage = FileStoreConfiguration(dirs=[os.path.dirname(__file__)], format="xml", extension="aiml", encoding="utf-8", delete_on_start=False)
+        self.configuration.client_configuration.storage._categories_storage = FileStoreConfiguration(dirs=[os.path.dirname(__file__)], format="xml", extension="aiml", encoding="utf-8", delete_on_start=False)
 
     def add_local_properties(self):
         client_context = self.create_client_context("console")

src/programy/clients/client.py

@@ -181,8 +181,8 @@ def parse_arguments(self, argument_parser):
         return client_args
 
     def load_license_keys(self):
-        if self.storage_factory.storage_engine_available(StorageFactory.LICENSE_KEYS) is True:
-            storage_engine = self.storage_factory.storage_engine(StorageFactory.LICENSE_KEYS)
+        if self.storage_factory.entity_storage_engine_available(StorageFactory.LICENSE_KEYS) is True:
+            storage_engine = self.storage_factory.entity_storage_engine(StorageFactory.LICENSE_KEYS)
             keys_store = storage_engine.license_store()
             keys_store.load(self._license_keys)
         else:

src/programy/config/bot/conversations.py

@@ -18,26 +18,16 @@
 from programy.utils.logging.ylogger import YLogger
 
 from programy.config.base import BaseConfigurationData
-from programy.dialog.storage.factory import ConversationStorageFactory
 
 class BotConversationsConfiguration(BaseConfigurationData):
 
     def __init__(self):
         BaseConfigurationData.__init__(self, name="conversations")
-        self._save = False
-        self._load = False
         self._max_histories = 100
         self._restore_last_topic = False
         self._initial_topic = "*"
         self._empty_on_start = False
-
-    @property
-    def save(self):
-        return self._save
-
-    @property
-    def load(self):
-        return self._load
+        self._multi_client = False
 
     @property
     def max_histories(self):
@@ -55,30 +45,31 @@ def restore_last_topic(self):
     def empty_on_start(self):
         return self._empty_on_start
 
+    @property
+    def multi_client(self):
+        return self._multi_client
+
     def load_config_section(self, configuration_file, configuration, bot_root):
         Conversations = configuration_file.get_section(self._section_name, configuration)
         if Conversations is not None:
-            self._save = configuration_file.get_bool_option(Conversations, "save", missing_value=True)
-            self._load = configuration_file.get_bool_option(Conversations, "load", missing_value=False)
             self._max_histories = configuration_file.get_int_option(Conversations, "max_histories", missing_value=100)
             self._initial_topic = configuration_file.get_option(Conversations, "initial_topic", missing_value="*")
             self._restore_last_topic = configuration_file.get_bool_option(Conversations, "restore_last_topic", missing_value=False)
             self._empty_on_start = configuration_file.get_bool_option(Conversations, "empty_on_start", missing_value=False)
+            self._multi_client = configuration_file.get_bool_option(Conversations, "multi_client", missing_value=False)
         else:
             YLogger.warning(self, "'Conversations' section missing from bot config, using defaults")
 
     def to_yaml(self, data, defaults=True):
         if defaults is True:
-            data['save'] = True
-            data['load'] = False
             data['max_histories'] = 100
             data['restore_last_topic'] = True
             data['initial_topic'] = "*"
             data['empty_on_start'] = True
+            data['multi_client'] = False
         else:
-            data['save'] = self._save
-            data['load'] = self._load
             data['max_histories'] = self._max_histories
             data['restore_last_topic'] = self._restore_last_topic
             data['initial_topic'] = self._initial_topic
             data['empty_on_start'] = self._empty_on_start
+            data['multi_client'] = self._multi_client

src/programy/config/brain/debugfiles.py

@@ -32,14 +32,15 @@ def __init__(self):
     def save_errors(self):
         return self._save_errors
 
+    @property
     def save_duplicates(self):
         return self._save_duplicates
 
     def load_config_section(self, configuration_file, configuration, bot_root):
-        binaries = configuration_file.get_section("defaults", configuration)
-        if binaries is not None:
-            self._save_errors = configuration_file.get_bool_option(binaries, "save-errors", missing_value=False)
-            self._save_duplicates = configuration_file.get_bool_option(binaries, "save-duplicates", missing_value=False)
+        debugfiles = configuration_file.get_section("debugfiles", configuration)
+        if debugfiles is not None:
+            self._save_errors = configuration_file.get_bool_option(debugfiles, "save-errors", missing_value=False)
+            self._save_duplicates = configuration_file.get_bool_option(debugfiles, "save-duplicates", missing_value=False)
         else:
             YLogger.warning(self, "'debugfiles' section missing from brain config, using debugfile defaults")
 

src/programy/context.py

@@ -94,4 +94,13 @@ def __str__(self):
             self._bot.id if self._bot else "",
             self._brain.id if self._brain else "",
             self._question_depth
-        )
+        )
+
+    def to_json(self):
+        return {
+            "clientid": self._client.id,
+            "userid": self._userid,
+            "botid": self._bot.id if self._bot else None,
+            "brainid": self._brain.id if self._brain else None,
+            "depth": self._question_depth
+        }