make cron a background loop, make cron have own connection, start imp…

…lementing pooling

config.py

@@ -19,6 +19,7 @@
     "cache_level_strs": True,
     "lang": "en",
     "gdpysbot_enabled": True,
+    "cron_delay": 20, # In minutes.
     "cheatless": {
         "enabled": True,
         "minimum_attempts_extreme_demon": 100,

conn/mysql.py

@@ -4,17 +4,27 @@
 class Myconn:
     def __init__(self):
         self.conn = None
+        self.pool = None
 
 
 myconn = Myconn()
 
 
 async def create_connection(loop, config: dict):
-    myconn.conn = await aiomysql.connect(
+    # Create the pool.
+    myconn.pool = await aiomysql.create_pool(
         host=config["sql_server"],
         port=3306,
         user=config["sql_user"],
         password=config["sql_password"],
         db=config["sql_db"],
         loop=loop,
     )
+
+    # Create the main connection based on the pool (for compatibility reasons)
+    myconn.conn = await myconn.pool.acquire()
+
+def conn_cleanup(conn: aiomysql.Connection) -> None:
+    """Closes the connection and cleans up after it."""
+    conn.close()
+    myconn.pool.release(conn)

cron/cachelb.py

@@ -1,36 +1,35 @@
-from conn.mysql import myconn
 from helpers.userhelper import user_helper
 
 # Lists of user objects in order of stars
 top_stars = []
 top_cp = []
 
 
-async def get_list(order: str = "stars") -> list:
+async def get_list(conn, order: str = "stars") -> list:
     """Gets list of users ordered desc by arg."""
-    async with myconn.conn.cursor() as mycursor:
+    async with conn.cursor() as mycursor:
         await mycursor.execute(
             f"SELECT extID FROM users WHERE isBanned = 0 ORDER BY {order} DESC LIMIT 100"
         )  # TODO: isBanned alternative
         list_id = await mycursor.fetchall()
     return [i[0] for i in list_id]
 
 
-async def cron_top_stars():
+async def cron_top_stars(conn):
     """Caches top 100 leaderboards."""
     top_stars.clear()
     # We have to cache all the user objects.
-    for account_id in await get_list("stars"):
+    for account_id in await get_list(conn, "stars"):
         top_stars.append(
             await user_helper.get_object(account_id)
         )  # Honestly I have no clue how to not store the objects separately twice plus this helps when adding features like lb freezing.
 
 
-async def cron_top_cp():
+async def cron_top_cp(conn):
     """Caches top cp leaderboards."""
     top_cp.clear()
     # We have to cache all the user objects.
-    for account_id in await get_list("creatorPoints"):
+    for account_id in await get_list(conn, "creatorPoints"):
         top_cp.append(
             await user_helper.get_object(account_id)
         )  # Honestly I have no clue how to not store the objects separately twice plus this helps when adding features like lb freezing.

cron/cachempgauntlets.py

@@ -1,5 +1,4 @@
 # Couldn't think of a good name for the file but this just caches the gauntlets and map-packs. Bunched them up as they are REALLY similar.
-from conn.mysql import myconn
 from objects.levels import Gauntlet, MapPack
 from objects.misc import RGB
 from helpers.lang import lang
@@ -9,11 +8,11 @@
 gauntlets = []
 
 
-async def cron_cache_mappacks():
+async def cron_cache_mappacks(conn):
     """Cron job that caches the map packs."""
     map_packs.clear()
     # We get all of the map packs from the database.
-    async with myconn.conn.cursor() as mycursor:
+    async with conn.cursor() as mycursor:
         await mycursor.execute(
             "SELECT ID, name, levels, stars, coins, difficulty, rgbcolors FROM mappacks"
         )  # Shouldnt be too much I think.
@@ -33,11 +32,11 @@ async def cron_cache_mappacks():
         )
 
 
-async def cron_cache_gauntlets():
+async def cron_cache_gauntlets(conn):
     """Caches the in-game gauntlets."""
     gauntlets.clear()
     # Getting all of the gauntlets from the database.
-    async with myconn.conn.cursor() as mycursor:
+    async with conn.cursor() as mycursor:
         await mycursor.execute(
             "SELECT ID, level1,level2,level3,level4,level5 FROM gauntlets"
         )

cron/cpcalc.py

@@ -1,4 +1,3 @@
-from conn.mysql import myconn
 from helpers.generalhelper import create_offsets_from_page, dict_keys
 from helpers.userhelper import user_helper
 from helpers.lang import lang
@@ -8,14 +7,14 @@
 PAGE_SIZE = 100  # You may make it larger to speed things up if you have a lot of free memory. Default should be fine.
 
 
-async def cron_calc_cp():
+async def cron_calc_cp(conn):
     """Server CP calculation."""
     # So I may make this more GDPyS style (using all of the level objects etc) but in this case, speed matters
     # a LOT more than usual. I have found this way to be MUCH faster and therefore decided to implement it as such here.
     cp_users = (
         {}
     )  # Dict (with keys being account IDs) that store user's final CP count.
-    async with myconn.conn.cursor() as mycursor:
+    async with conn.cursor() as mycursor:
         # We wipe the whole server's CP values in case a person lost all their levels ratings (as he won't be recalculated for efficiency reasons).
         await mycursor.execute("UPDATE users SET creatorPoints = 0")
         # Now we get all the levels that would give cp.

cron/cron.py

@@ -8,7 +8,11 @@
 import logging
 import traceback
 import asyncio
-import conn
+from conn.mysql import (
+    myconn,
+    create_connection,
+    conn_cleanup
+)
 
 CRON_JOBS = [  #
     cron_calc_ranks,
@@ -26,26 +30,39 @@ async def run_cron():
         loop = asyncio.get_event_loop()
         load_config()
         lang.load_langs()
-        await conn.mysql.create_connection(loop, user_config)
+        await create_connection(loop, user_config)
     total_t = Timer()
     total_t.start()
+    conn = await myconn.pool.acquire()
     for job in CRON_JOBS:
         logging.debug(lang.debug("cron_job_running", job.__name__))
         t = Timer()
         t.start()
         try:
-            await job()
+            await job(conn)
         except Exception as e:
             logging.error(lang.error("CRON_FAIL", job.__name__, e))
             logging.debug(traceback.format_exc())
         # So we don't get 32846238746238ms or 0.0s
         t_str = time_str(t)
         logging.info(lang.info("CRON_FINISH", job.__name__, t_str))
 
+    # Close the connection and free the pool.
+    conn_cleanup(conn)
+
     # Don't copy paste code now watch me not follow my own advice. If I have to use this somewhere else, I will move this to timehelper.
     t_str = time_str(total_t)
     logging.info(lang.info("CRON_ALL_FINISH", t_str))
 
+async def cron_loop():
+    """Creates a loop for cron to run every x minutes."""
+    logging.debug(lang.debug("cron_loop_start", user_config["cron_delay"]))
+    while True:
+        await run_cron()
+        # Compute it here in case of a config reload.
+        loop_delay = user_config["cron_delay"] * 60 # Time between cron job in minutes.
+        logging.debug(lang.debug("cron_next_due", user_config["cron_delay"]))
+        await asyncio.sleep(loop_delay)
 
 # async def cron_gather():
 #    """An experimental way of running all of the cron jobs simultaniously async style."""

cron/rankcalc.py

@@ -1,21 +1,17 @@
-from conn.mysql import myconn
 from constants import Permissions
 
 ranks = {}  # For now I will store it here.
 
 
-async def cron_calc_ranks() -> None:
+async def cron_calc_ranks(conn) -> None:
     """Calculates all ranks for users and stores them in cache."""  # I may move this to a cron category however that does not currently exist.
-    async with myconn.conn.cursor() as mycursor:
+    async with conn.cursor() as mycursor:
         await mycursor.execute(
             "SELECT extID FROM users WHERE extID IN (SELECT accountID FROM accounts WHERE privileges & %s AND isBot = 0) ORDER BY stars DESC",
             (Permissions.AUTH,),
         )
         users = await mycursor.fetchall()
 
-    curr_rank = (
-        0  # There is most likely a better way to do this but I don't know it yet
-    )
-    for user in users:
-        curr_rank += 1
-        ranks[int(user[0])] = curr_rank
+    for rank, user in enumerate(users):
+        # Add 1 to rank as lists are 0 indexed.
+        ranks[int(user[0])] = rank + 1

lang/en.json

@@ -42,7 +42,9 @@
     "no_score" : "This user has not set a score on this level.",
     "overwrite_score" : "Overwriting this score.",
     "adding_handler" : "Adding handler: {0} | {1}",
-    "message_not_found" : "The message with the ID of {0} could not be found! It either does not exist or the user lacks the sufficient privileges to access it."
+    "message_not_found" : "The message with the ID of {0} could not be found! It either does not exist or the user lacks the sufficient privileges to access it.",
+    "cron_loop_start": "Cron loop started. Set to run every {0} minutes.",
+    "cron_next_due": "The next cron job will run in {0} minutes."
   },
   "runtime": {
     "shutdown": "Shutting down! Bye!",

main.py

@@ -49,7 +49,7 @@
 from helpers.priveliegehelper import priv_helper
 from helpers.lang import lang
 from helpers.generalhelper import time_coro
-from cron.cron import run_cron
+from cron.cron import cron_loop
 from constants import ASCII_ART, Colours
 from conn.mysql import create_connection
 from os import path
@@ -168,8 +168,10 @@ async def init(loop):
     app = web.Application(loop=loop)
     await create_connection(loop, user_config)
     await priv_helper.cache_privs()
-    await run_cron()
+    # await run_cron()
     # await cron_gather()
+    # Make cron loop in the background.
+    asyncio.create_task(cron_loop())
     songs.top_artists = await songs._top_artists()
     # Setting up rate limiter
     rate_limiter.add_to_struct(