Fix 404/500 issues on tools/scummvm for both systems and images

helpers.py

@@ -41,6 +41,10 @@ def list_files_with_extensions(path, extensions):
 def file_exists(path, file):
     return os.path.isfile(os.path.join(path, file))
 
+def file_get_size(file):
+    if os.path.isfile(file):
+        return os.path.getsize(file)
+
 def folder_exists(path):
     return os.path.isdir(path)
 
@@ -49,6 +53,12 @@ def remove_prefix(text, prefix):
         return text[len(prefix):]
     return text
 
+def find(node, look):
+    if node.find(look) != None and node.find(look).text != None:
+        return node.find(look).text
+    else:
+        return False
+
 def find_normalized(node, look):
     if node.find(look) != None and node.find(look).text != None:
         return fix_text(unescape(node.find(look).text))
@@ -69,24 +79,21 @@ def find_int(node, look):
 
 def find_image_path(node, look):
     if node.find(look) != None and node.find(look).text != None:
-        return remove_prefix(node.find(look).text, "./images/")
+        return node.find(look).text
     else:
         return False
 
 def find_video_path(node, look):
     if node.find(look) != None and node.find(look).text != None:
-        return remove_prefix(node.find(look).text, "./videos/")
+        return node.find(look).text
     else:
         return False
 
 def find_normalized_path(node, look):
     return remove_prefix(find_normalized(node, look), "./")
 
-def find_normalized_system_path(node, look):
-    return remove_prefix(find_normalized(node, look), roms_folder)
-
 def system_path(system, *paths):
-    return os.path.join(os.path.abspath(roms_folder), system, *paths)
+    return os.path.join(os.path.abspath(get_system_path(system)), *paths)
 
 def find_date(node, look):
     if find_normalized(node, look):
@@ -101,11 +108,16 @@ def find_date_form(node, look):
         return False
 
 def format_xml_date(date):
-    return datetime.datetime.strptime(date, '%Y%m%dT%H%M%S').strftime('%d %B %Y')
+    try:
+      return datetime.datetime.strptime(date, '%Y%m%dT%H%M%S').strftime('%d %B %Y')
+    except ValueError:
+      return date
 
 def format_xml_date_form(date):
-    return datetime.datetime.strptime(date, '%Y%m%dT%H%M%S').strftime('%Y-%m-%d')
-
+    try:
+      return datetime.datetime.strptime(date, '%Y%m%dT%H%M%S').strftime('%Y-%m-%d')
+    except ValueError:
+      return date
 def date_to_xml(date):
     return datetime.datetime.strptime(date, '%Y-%m-%d').strftime('%Y%m%dT%H%M%S')
 
@@ -149,8 +161,7 @@ def getsize_fmt(path):
         return "0b"
 
 def map_system_folder(system):
-    es_systems = ElementTree.parse(es_systems_path).getroot()
-    system_ele = es_systems.find(".//system[path=\"%s%s\"]" % (roms_folder, system))
+    system_ele = get_system_element(system)
     if system_ele:
         return system_ele.find("fullname").text
     return system
@@ -203,8 +214,30 @@ def get_system_info(system):
         'extension': find_normalized(system, 'extension')
     }
 
+es_systems = None
+
+# This lazy caches the ES systems list (from es_systems.cfg)
+# - the file is read only - so caching is ok
+# - caching does trade a bit of memory usage for not having to reload from file.
+#   I think this is necessary to figure out the non-standard system paths. 
+#   - Another option to limit memory would be to refactor to just cache the system paths in a dictionary, etc.
+def get_systems_list():
+    global es_systems
+
+    if not es_systems:
+        es_systems = ElementTree.parse(es_systems_path).getroot()
+
+    return es_systems
+
+def get_system_element(system_name):
+    return get_systems_list().findall(".//system/[name=\"%s\"]" % ( system_name))[0]
+
+def get_system_path(system_name):
+    return find_normalized(get_system_element(system_name),'path')
+
 def get_game_info(system, game_ref):
-    gamelist = os.path.join(roms_folder, system, 'gamelist.xml')
+    system_folder_path = get_system_path(system)
+    gamelist = os.path.join(system_folder_path, 'gamelist.xml')
 
     if os.path.isfile(gamelist):
         root = ElementTree.parse(gamelist).getroot()
@@ -218,8 +251,9 @@ def get_game_info(system, game_ref):
 
         if ele != None:
 
-            rom_filename = remove_prefix(find_normalized(ele, 'path'), './')
-            rom_path = os.path.join(roms_folder, system, rom_filename)
+            #NOTE: is it important the rom_filename is not 'normalized' as it will mess up loading from file system in cases like: é
+            rom_filename = remove_prefix(find(ele, 'path'), './')
+            rom_path = os.path.join(system_folder_path, rom_filename)
 
             game = {
                 'id': ele.attrib.get('id', False) or rom_filename,
@@ -236,34 +270,34 @@ def get_game_info(system, game_ref):
                 'marquee': find_image_path(ele, 'marquee'),
                 'video': find_video_path(ele, 'video'),
                 'rating': find_float(ele, 'rating'),
-                'path': find_normalized_path(ele, 'path'),
+                'path': rom_filename,
                 'releasedate': find_date(ele, 'releasedate'),
                 'releasedate_form': find_date_form(ele, 'releasedate'),
                 'lastplayed': find_date(ele, 'lastplayed'),
                 'playcount': find_int(ele, 'playcount'),
                 'gametime': find_int(ele, 'gametime'),
                 'size': getsize_fmt(rom_path),
-                'size_raw': os.path.getsize(rom_path),
+                'size_raw': file_get_size(rom_path),
                 'have_rom': os.path.isfile(rom_path),
                 'saves': find_saves(system, rom_filename),
                 'screenshots': find_screenshots(rom_filename)
             }
 
             if game["image"]:
-                game["image_size"] = os.path.getsize(os.path.join(roms_folder, system, 'images', find_image_path(ele, 'image')))
+                game["image_size"] = file_get_size(os.path.join(system_folder_path, find_image_path(ele, 'image')))
 
             if game["thumbnail"]:
-                game["thumbnail_size"] = os.path.getsize(os.path.join(roms_folder, system, 'images', find_image_path(ele, 'thumbnail')))
+                game["thumbnail_size"] = file_get_size(os.path.join(system_folder_path, find_image_path(ele, 'thumbnail')))
 
             if game["marquee"]:
-                game["marquee_size"] = os.path.getsize(os.path.join(roms_folder, system, 'images', find_image_path(ele, 'marquee')))
+                game["marquee_size"] = file_get_size(os.path.join(system_folder_path, find_image_path(ele, 'marquee')))
 
             if game["video"]:
-                game["video_size"] = os.path.getsize(os.path.join(roms_folder, system, 'videos', find_video_path(ele, 'video')))
+                game["video_size"] = file_get_size(os.path.join(system_folder_path, find_image_path(ele, 'video')))
 
             return game
 
-    rom_path = os.path.join(roms_folder, system, unescape(game_ref))
+    rom_path = os.path.join(system_folder_path, unescape(game_ref))
     if os.path.isfile(rom_path):
         game = {
             'id': game_ref,
@@ -301,9 +335,9 @@ def get_game_info(system, game_ref):
     else:
         return False
 
-def list_roms(system_ele):
-    system = find_normalized_system_path(system_ele, 'path')
-    gamelist = os.path.join(roms_folder, system, 'gamelist.xml')
+def list_roms(system):
+    system_ele = get_system_element(system)
+    gamelist = os.path.join(get_system_path(system), 'gamelist.xml')
 
     roms = []
     known_roms = []
@@ -312,7 +346,14 @@ def list_roms(system_ele):
         root = ElementTree.parse(gamelist).getroot()
         for game in root.iter('game'):
             id = game.attrib.get('id')
-            rom_filename = normalize_path(find_normalized(game, 'path'))
+
+            # NOTE: it is important the filename is not normalized using 'find_normalized' or it will mess up loading from file system
+            rom_filename = remove_prefix(find(game, 'path'), './')
+
+            # Hide any entries where files don't exist
+            if not os.path.isfile(os.path.join(get_system_path(system),rom_filename)):
+                continue
+
             known_roms.append(rom_filename)
             roms.append({
                 'id': id or False,
@@ -326,7 +367,7 @@ def list_roms(system_ele):
                 'path': rom_filename,
             })
 
-    rom_files = list_files(os.path.join(roms_folder, system))
+    rom_files = list_files(get_system_path(system))
     extensions = system_ele.find("extension").text.split(" ")
 
     for file in rom_files:

server.py

@@ -15,16 +15,14 @@
 @view('index')
 def index():
     systems = []
-    es_systems = ElementTree.parse(es_systems_path).getroot()
 
-    for system in es_systems.iter('system'):
+    for system in get_systems_list().iter('system'):
         system_folder_path = find_normalized(system, 'path')
         extensions = find_normalized(system, 'extension').split(" ")
         systems.append({
             'fullname': find_normalized(system, 'fullname'),
             'name': find_normalized(system, 'name'),
             'manufacturer': find_image_path(system, 'manufacturer'),
-            'folder': find_normalized_system_path(system, 'path'),
             'path': find_normalized(system, 'path'),
             'roms': len(list_files_with_extensions(system_folder_path, extensions))
         })
@@ -37,11 +35,10 @@ def index():
 @route('/system/<system>')
 @view('system')
 def view_system(system):
-    es_systems = ElementTree.parse(es_systems_path).getroot()
-    system_ele = es_systems.findall(".//system/[path=\"%s%s\"]" % (roms_folder, system))[0]
+    system_ele = get_system_element(system)
     system_info = get_system_info(system_ele)
     system_name = system_info["fullname"] or system
-    games = list_roms(system_ele)
+    games = list_roms(system)
     sorted_games = sorted(games, key=lambda k: (k['name']))
     return dict(system=system, system_name=system_name, games=json.dumps(sorted_games), system_info=system_info)
 
@@ -59,8 +56,7 @@ def view_game(system, game_ref):
 @route('/edit/<system>/<game_ref:path>')
 def edit(system, game_ref):
     game = get_game_info(system, game_ref)
-    es_systems = ElementTree.parse(es_systems_path).getroot()
-    system_ele = es_systems.findall(".//system/[path=\"%s%s\"]" % (roms_folder, system))[0]
+    system_ele = get_system_element(system)
     system_info = get_system_info(system_ele)
     system_name = system_info["fullname"] or system
     extensions = set([ext.lower() for ext in system_info["extension"].split(" ")])
@@ -87,7 +83,7 @@ def edit(system, game_ref):
 
 @route('/exists/<system>/<rom>')
 def exists(system, rom):
-    gamelist = os.path.join(roms_folder, system, 'gamelist.xml')
+    gamelist = os.path.join(get_system_path(system), 'gamelist.xml')
 
     if os.path.isfile(gamelist):
         root = ElementTree.parse(gamelist).getroot()
@@ -103,7 +99,7 @@ def exists(system, rom):
 @post('/upload/<system>')
 def upload_rom(system):
     if request.forms.get('submit'):
-        gamelist = os.path.join(roms_folder, system, 'gamelist.xml')
+        gamelist = os.path.join(get_system_path(system), 'gamelist.xml')
 
         if os.path.isfile(gamelist):
             tree = ElementTree.parse(gamelist)
@@ -138,12 +134,18 @@ def upload_rom(system):
                 game = root.find(".//game/[path=\"./%s\"]" % rom.raw_filename)
             else:
                 game = SubElement(root, 'game', attrib={ 'id': str(uuid4()) })
+                game.tail = "\n"
                 entry = SubElement(game, 'path')
                 entry.text = "./%s" % rom.raw_filename
             rom_filename = rom.raw_filename
         elif existing_rom:
             game = root.find(".//game/[path=\"./%s\"]" % existing_rom)
             rom_filename = existing_rom
+            if not game:
+                game = SubElement(root, 'game', attrib={ 'id': str(uuid4()) })
+                game.tail = "\n"
+                entry = SubElement(game, 'path')
+                entry.text = "./%s" % existing_rom
         else:
             return redirect('/')
 
@@ -181,18 +183,22 @@ def upload_rom(system):
 
         if marquee and marquee.filename != 'empty':
             update_game_entry(game, 'marquee', "./images/%s" % marquee.raw_filename)
+            os.makedirs(system_path(system, 'images'), exist_ok=True)
             marquee.save(system_path(system, 'images', marquee.raw_filename), overwrite=True)
 
         if screenshot and screenshot.filename != 'empty':
             update_game_entry(game, 'image', "./images/%s" % screenshot.raw_filename)
+            os.makedirs(system_path(system, 'images'), exist_ok=True)
             screenshot.save(system_path(system, 'images', screenshot.raw_filename), overwrite=True)
 
         if boxart and boxart.filename != 'empty':
             update_game_entry(game, 'thumbnail', "./images/%s" % boxart.raw_filename)
+            os.makedirs(system_path(system, 'images'), exist_ok=True)
             boxart.save(system_path(system, 'images', boxart.raw_filename), overwrite=True)
 
         if video and video.filename != 'empty':
             update_game_entry(game, 'video', "./videos/%s" % boxart.raw_filename)
+            os.makedirs(system_path(system, 'videos'), exist_ok=True)
             video.save(system_path(system, 'videos', boxart.raw_filename), overwrite=True)
 
         with open(gamelist, 'wb') as file:
@@ -202,8 +208,7 @@ def upload_rom(system):
 
         return redirect('/system/%s/%s' % (system, game_id if game_id else rom_filename))
     else:
-        es_systems = ElementTree.parse(es_systems_path).getroot()
-        system_ele = es_systems.findall(".//system/[path=\"%s%s\"]" % (roms_folder, system))[0]
+        system_ele = get_system_element(system)
         system_info = get_system_info(system_ele)
         system_name = system_info["fullname"] or system
         extensions = set([ext.lower() for ext in system_info["extension"].split(" ")])
@@ -231,26 +236,26 @@ def view_svg(text):
     return template('views/empty_svg.tpl', system=text.split())
 
 
-@route('/image/<system>/<image>')
+@route('/image/<system>/<image:path>')
 def view_image(system, image):
-    image = os.path.join('images', unquote(image))
-    path = os.path.join(roms_folder, system)
+    image = unquote(image)
+    path=get_system_path(system)
     response.set_header('Cache-Control', 'max-age=3600')
     return static_file(image, root=path)
 
 
-@route('/video/<system>/<video>')
+@route('/video/<system>/<video:path>')
 def view_video(system, video):
-    video = os.path.join('videos', unquote(video))
-    path = os.path.join(roms_folder, system)
+    video = unquote(video)
+    path=get_system_path(system)
     response.set_header('Cache-Control', 'max-age=3600')
     return static_file(video, root=path)
 
 
-@route('/rom/<system>/<rom>')
+@route('/rom/<system>/<rom:path>')
 def download_rom(system, rom):
     rom = unquote(rom)
-    path = os.path.join(roms_folder, system)
+    path=get_system_path(system)
     return static_file(rom, root=path, download=rom)
 
 
@@ -282,7 +287,7 @@ def assets(path):
 
 @route('/launch/<system>/<rom>')
 def assets(system, rom):
-    path = os.path.join(roms_folder, system, rom)
+    path = os.path.join(get_system_path(system), rom)
     start_game(path)
 
 

views/index.tpl

@@ -1,7 +1,7 @@
 % include('_header.tpl', nav={ "Systems": "/" }, search=True)
     <ul role="list" class="grid grid-cols-2 gap-5 sm:gap-6 sm:grid-cols-3 lg:grid-cols-4">
         <template x-for="item in filteredData" :key="item.name">
-            <a :href="`/system/${item.folder}`">
+            <a :href="`/system/${item.name}`">
                 <li class="col-span-1 shadow dark:shadow-none rounded-md" :class="item.roms == 0 ? 'opacity-60' : ''">
                     <div class="flex items-center justify-center w-100 h-20 bg-theme-700 dark:bg-theme-800 text-white text-sm font-medium rounded-t-md">
                         <img loading="lazy" :src="`/svg/${item.name}`" class="w-1/2 max-h-20 p-2" :alt="item.fullname">