Merge pull request #357 from zurdi15/scheduled-tasks

Scheduled tasks

.gitignore

@@ -52,3 +52,7 @@ backend/romm_test/logs
 
 # service worker
 frontend/dev-dist
+
+# outside data
+switch_titledb.json
+mame.xml

backend/config/__init__.py

@@ -66,3 +66,29 @@
 ROMM_AUTH_SECRET_KEY: Final = os.environ.get(
     "ROMM_AUTH_SECRET_KEY", secrets.token_hex(32)
 )
+
+# TASKS
+ENABLE_RESCAN_ON_FILESYSTEM_CHANGE: Final = (
+    os.environ.get("ENABLE_RESCAN_ON_FILESYSTEM_CHANGE", "false") == "true"
+)
+RESCAN_ON_FILESYSTEM_CHANGE_DELAY: Final = int(
+    os.environ.get("RESCAN_ON_FILESYSTEM_CHANGE_DELAY", 5)  # 5 minutes
+)
+ENABLE_SCHEDULED_RESCAN: Final = (
+    os.environ.get("ENABLE_SCHEDULED_RESCAN", "false") == "true"
+)
+SCHEDULED_RESCAN_CRON: Final = os.environ.get(
+    "SCHEDULED_RESCAN_CRON", "0 3 * * *"  # At 3:00 AM every day
+)
+ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB: Final = (
+    os.environ.get("ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB", "false") == "true"
+)
+SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: Final = os.environ.get(
+    "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON", "0 4 * * *"  # At 4:00 AM every day
+)
+ENABLE_SCHEDULED_UPDATE_MAME_XML: Final = (
+    os.environ.get("ENABLE_SCHEDULED_UPDATE_MAME_XML", "false") == "true"
+)
+SCHEDULED_UPDATE_MAME_XML_CRON: Final = os.environ.get(
+    "SCHEDULED_UPDATE_MAME_XML_CRON", "0 4 * * *"  # At 4:00 AM every day
+)

backend/endpoints/scan.py

@@ -1,22 +1,21 @@
 import emoji
 import socketio  # type: ignore
-from rq import Queue
 
 from logger.logger import log
 from utils import fs, fastapi
 from exceptions.fs_exceptions import PlatformsNotFoundException, RomsNotFoundException
 from handler import dbh
 from utils.socket import socket_server
-from utils.cache import redis_client, redis_url
+from utils.redis import high_prio_queue, redis_url
 from endpoints.platform import PlatformSchema
 from endpoints.rom import RomSchema
 from config import ENABLE_EXPERIMENTAL_REDIS
 
-scan_queue = Queue(connection=redis_client)
-
 
 async def scan_platforms(
-    platform_slugs: list[str], complete_rescan: bool, selected_roms: list[str]
+    platform_slugs: list[str],
+    complete_rescan: bool = False,
+    selected_roms: list[str] = (),
 ):
     # Connect to external socketio server
     sm = (
@@ -57,7 +56,7 @@ async def scan_platforms(
             if rom_id and rom_id not in selected_roms and not complete_rescan:
                 continue
 
-            scanned_rom = fastapi.scan_rom(scanned_platform, fs_rom)
+            scanned_rom = await fastapi.scan_rom(scanned_platform, fs_rom)
             if rom_id:
                 scanned_rom.id = rom_id
 
@@ -90,7 +89,7 @@ async def scan_handler(_sid: str, options: dict):
 
     # Run in worker if redis is available
     if ENABLE_EXPERIMENTAL_REDIS:
-        return scan_queue.enqueue(
+        return high_prio_queue.enqueue(
             scan_platforms,
             platform_slugs,
             complete_rescan,

backend/endpoints/tests/test_heartbeat.py

@@ -8,4 +8,14 @@
 def test_heartbeat():
     response = client.get("/heartbeat")
     assert response.status_code == 200
-    assert response.json() == {"ROMM_AUTH_ENABLED": True}
+    assert response.json() == {
+        'ROMM_AUTH_ENABLED': True,
+        'ENABLE_RESCAN_ON_FILESYSTEM_CHANGE': True,
+        'ENABLE_SCHEDULED_RESCAN': True,
+        'ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB': True,
+        'ENABLE_SCHEDULED_UPDATE_MAME_XML': True,
+        'RESCAN_ON_FILESYSTEM_CHANGE_DELAY': 5,
+        'SCHEDULED_RESCAN_CRON': '0 3 * * *',
+        'SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON': '0 4 * * *',
+        'SCHEDULED_UPDATE_MAME_XML_CRON': '0 4 * * *',
+    }

backend/handler/ps2_opl_index.py → backend/handler/fixtures/ps2_opl_index.json

@@ -1,4 +1,4 @@
-opl_index = {
+{
   "SLES_556.71": {
     "Name": "Fifa '14",
     "Region": "PAL"

backend/handler/igdb_handler.py

@@ -4,6 +4,9 @@
 import requests
 import re
 import time
+import os
+import json
+import xmltodict
 from unidecode import unidecode as uc
 from requests.exceptions import HTTPError, Timeout
 from typing import Final
@@ -12,13 +15,27 @@
 from utils import get_file_name_with_no_tags as get_search_term
 from logger.logger import log
 from utils.cache import cache
-from .ps2_opl_index import opl_index
+from tasks.update_switch_titledb import update_switch_titledb_task
+from tasks.update_mame_xml import update_mame_xml_task
 
 MAIN_GAME_CATEGORY: Final = 0
 EXPANDED_GAME_CATEGORY: Final = 10
 N_SCREENSHOTS: Final = 5
 PS2_IGDB_ID: Final = 8
+SWITCH_IGDB_ID: Final = 130
+ARCADE_IGDB_ID: Final = 52
+
 PS2_OPL_REGEX: Final = r"^([A-Z]{4}_\d{3}\.\d{2})\..*$"
+PS2_OPL_INDEX_FILE: Final = os.path.join(
+    os.path.dirname(__file__), "fixtures", "ps2_opl_index.json"
+)
+
+SWITCH_TITLEDB_REGEX: Final = r"^(70[0-9]{12})$"
+SWITCH_TITLEDB_INDEX_FILE: Final = os.path.join(
+    os.path.dirname(__file__), "fixtures", "switch_titledb.json"
+)
+
+MAME_XML_FILE: Final = os.path.join(os.path.dirname(__file__), "fixtures", "mame.xml")
 
 
 class IGDBHandler:
@@ -141,16 +158,60 @@ def get_platform(self, p_slug: str):
         }
 
     @check_twitch_token
-    def get_rom(self, file_name: str, p_igdb_id: int):
+    async def get_rom(self, file_name: str, p_igdb_id: int):
         search_term = get_search_term(file_name)
 
         # Patch support for PS2 OPL flename format
         match = re.match(PS2_OPL_REGEX, search_term)
         if p_igdb_id == PS2_IGDB_ID and match:
             serial_code = match.group(1)
-            index_entry = opl_index.get(serial_code, None)
-            if index_entry:
-                search_term = index_entry["Name"]  # type: ignore
+
+            with open(PS2_OPL_INDEX_FILE, "r") as index_json:
+                opl_index = json.loads(index_json.read())
+                index_entry = opl_index.get(serial_code, None)
+                if index_entry:
+                    search_term = index_entry["Name"]  # type: ignore
+
+        # Patch support for switch titleID filename format
+        match = re.match(SWITCH_TITLEDB_REGEX, search_term)
+        if p_igdb_id == SWITCH_IGDB_ID and match:
+            title_id = match.group(1)
+            titledb_index = {}
+
+            try:
+                with open(SWITCH_TITLEDB_INDEX_FILE, "r") as index_json:
+                    titledb_index = json.loads(index_json.read())
+            except FileNotFoundError:
+                log.warning("Fetching the Switch titleDB index file...")
+                await update_switch_titledb_task.run(force=True)
+
+                with open(SWITCH_TITLEDB_INDEX_FILE, "r") as index_json:
+                    titledb_index = json.loads(index_json.read())
+            finally:
+                index_entry = titledb_index.get(title_id, None)
+                if index_entry:
+                    search_term = index_entry["name"]  # type: ignore
+
+        if p_igdb_id == ARCADE_IGDB_ID:
+            mame_index = {}
+
+            try:
+                with open(MAME_XML_FILE, "r") as index_xml:
+                    mame_index = xmltodict.parse(index_xml.read())
+            except FileNotFoundError:
+                log.warning("Fetching the MAME XML file from HyperspinFE...")
+                await update_mame_xml_task.run(force=True)
+
+                with open(MAME_XML_FILE, "r") as index_xml:
+                    mame_index = xmltodict.parse(index_xml.read())
+            finally:
+                index_entry = [
+                    game
+                    for game in mame_index["menu"]["game"]
+                    if game["@name"] == search_term
+                ]
+                if index_entry:
+                    search_term = index_entry[0].get("description", search_term)
 
         res = (
             self._search_rom(uc(search_term), p_igdb_id, MAIN_GAME_CATEGORY)