Merge pull request #99 from LanceMaverick/improve-paginator-speed

Improve paginator speed

.gitignore

@@ -1,4 +1,5 @@
 # Custom
+*.dbbin
 beards/*/config.yml
 config.py
 .#*

config.py.example

@@ -20,6 +20,7 @@ beards = [
 ]
 
 db_url = "sqlite:///skybeard-2.db"
+db_bin_path = "./db_binary_entries"
 
 #put yourself, and anyone else you wish to, as an admin
 admins = [

main.py

@@ -198,6 +198,10 @@ def main(config):
     # TODO consider making this not a parrt of the BeardChatHandler class now
     # that we use pyconfig.
     BeardChatHandler.setup_beards(parsed.key, config.db_url)
+    pyconfig.set('db_url', config.db_url)
+    pyconfig.set('db_bin_path', config.db_bin_path)
+    if not os.path.exists(pyconfig.get('db_bin_path')):
+        os.mkdir(pyconfig.get('db_bin_path'))
 
     # If the user does not specially request --no-help, set up help command.
     if not parsed.no_help:

skybeard/bearddbtable.py

@@ -1,3 +1,6 @@
+import string
+import random
+import os
 import logging
 import dataset
 
@@ -14,8 +17,9 @@ class BeardDBTable(object):
     """
     def __init__(self, beard, table_name, **kwargs):
         self.beard_name = type(beard).__name__
-        self.table_name = "{}_{}".format(
+        self.table_name = "{}_{}_{}".format(
             self.beard_name,
+            beard.chat_id,
             table_name
         )
         self.kwargs = kwargs
@@ -41,3 +45,37 @@ def __getattr__(self, name):
         except AttributeError:
             raise AttributeError(
                 "Open table not found. Are you using BeardDBTable with with?")
+
+
+# async def get_binary_entry(path):
+#     return open(os.path.join(pyconfig.get('db_bin_path'), path), 'rb')
+
+
+async def make_binary_entry_filename(table, key):
+    # Assume the random string has been found, until it's not been found.
+    random_string_found = True
+    while random_string_found:
+        random_string = "".join([random.choice(string.ascii_letters) for x in range(50)])
+        for d in os.listdir(pyconfig.get('db_bin_path')):
+            if random_string in d:
+                break
+        else:
+            random_string_found = False
+
+    primary_key = "_".join(table.table.table.primary_key.columns.keys())
+
+    return os.path.join(
+        pyconfig.get('db_bin_path'), "{}_{}_{}_{}.dbbin".format(
+            table.table_name,
+            primary_key,
+            key,
+            random_string))
+
+
+# async def create_binary_entry(table, key, bytes_object):
+#     object_path = os.path.join(
+#         pyconfig.get('db_bin_path'),
+#         binary_entry_filename(table, key))
+#     with open(object_path, 'wb') as f:
+#         f.write()
+#     return object_path

skybeard/mixins.py

@@ -1,9 +1,13 @@
+import logging
+
 import dill
 from telepot import glance, message_identifier
 from telepot.namedtuple import InlineKeyboardMarkup, InlineKeyboardButton
 
 from .beards import ThatsNotMineException
-from .bearddbtable import BeardDBTable
+from .bearddbtable import BeardDBTable, make_binary_entry_filename
+
+logger = logging.getLogger(__name__)
 
 
 class PaginatorMixin:
@@ -41,7 +45,7 @@ async def __make_prev_next_keyboard(self, prev_seq, next_seq):
     async def send_paginated_message(
             self,
             next_seq,
-            formatter,
+            formatter_func,
             curr_item=None,
             prev_seq=None,
     ):
@@ -66,19 +70,19 @@ async def send_paginated_message(
 
         keyboard = await self.__make_prev_next_keyboard(prev_seq, next_seq)
         sent_msg = await self.sender.sendMessage(
-            await formatter(curr_item),
+            await formatter_func(curr_item),
             parse_mode='HTML',
             reply_markup=keyboard
         )
 
         with self._paginator_table as table:
             entry_to_insert = {
                 'message_id': sent_msg['message_id'],
-                'prev_seq': dill.dumps(prev_seq),
-                'curr_item': dill.dumps(curr_item),
-                'next_seq': dill.dumps(next_seq),
-                'formatter_func': dill.dumps(formatter)
             }
+            for binary_name in ['prev_seq', 'curr_item', 'next_seq', 'formatter_func']:
+                entry_to_insert[binary_name] = await make_binary_entry_filename(table, binary_name)
+                dill.dump(locals()[binary_name], open(entry_to_insert[binary_name], 'wb'))
+
             table.insert(entry_to_insert)
 
     async def on_callback_query(self, msg):
@@ -96,36 +100,56 @@ async def on_callback_query(self, msg):
                 self.logger.debug("Got entry for message id: {}".format(
                     entry['message_id']))
 
-                prev_seq = dill.loads(entry['prev_seq'])
-                curr_item = dill.loads(entry['curr_item'])
-                next_seq = dill.loads(entry['next_seq'])
+                logger.debug("Loading prev_seq...")
+                prev_seq = dill.load(open(entry['prev_seq'], 'rb'))
+                logger.debug("Loading curr_item...")
+                curr_item = dill.load(open(entry['curr_item'], 'rb'))
+                logger.debug("Loading next_seq...")
+                next_seq = dill.load(open(entry['next_seq'], 'rb'))
+                logger.debug("Loading formatter_func...")
+                formatter_func = dill.load(open(entry['formatter_func'], 'rb'))
+                logger.debug("Un-dill-ed all objects.")
 
                 if data == 'p':
+                    logger.debug("Getting previous item.")
                     next_seq.insert(0, curr_item)
                     curr_item = prev_seq[-1]
                     prev_seq = prev_seq[:-1]
                 if data == 'n':
+                    logger.debug("Getting next item.")
                     prev_seq.append(curr_item)
                     curr_item = next_seq[0]
                     next_seq = next_seq[1:]
 
-                entry['prev_seq'] = dill.dumps(prev_seq)
-                entry['curr_item'] = dill.dumps(curr_item)
-                entry['next_seq'] = dill.dumps(next_seq)
-                with self._paginator_table as table:
-                    table.update(entry, ['message_id'])
-
                 keyboard = await self.__make_prev_next_keyboard(
                     prev_seq, next_seq)
 
-                formatter_func = dill.loads(entry['formatter_func'])
-
                 await self.bot.editMessageText(
                     message_identifier(msg['message']),
                     await formatter_func(curr_item),
                     parse_mode='HTML',
                     reply_markup=keyboard
                 )
+
+                logger.debug("Dumping prev_seq...")
+                dill.dump(prev_seq, open(entry['prev_seq'], 'wb'))
+                logger.debug("Dumping curr_item...")
+                dill.dump(curr_item, open(entry['curr_item'], 'wb'))
+                logger.debug("Dumping next_seq...")
+                dill.dump(next_seq, open(entry['next_seq'], 'wb'))
+                logger.debug("dill-ed all objects.")
+
+                # logger.debug("Updating prev_seq entry.")
+                # entry['prev_seq'] = dill.dumps(prev_seq)
+                # logger.debug("Updating curr_item entry.")
+                # entry['curr_item'] = dill.dumps(curr_item)
+                # logger.debug("Updating next_seq entry.")
+                # entry['next_seq'] = dill.dumps(next_seq)
+                # logger.debug("Updating entry in database.")
+                # with self._paginator_table as table:
+                #     table.update(entry, ['message_id'])
+                # logger.debug("Entry updated.")
+
         except ThatsNotMineException:
             pass