Merge pull request #478 from zurdi15/romm-473

[ROMM-473] Enhanced region and language tagging

backend/alembic/versions/0012_add_regions_languages.py

@@ -0,0 +1,44 @@
+"""empty message
+
+Revision ID: 0012_add_regions_languages
+Revises: 0011_drop_has_cover
+Create Date: 2023-12-03 10:54:46.859106
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = '0012_add_regions_languages'
+down_revision = '0011_drop_has_cover'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('roms', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('regions', sa.JSON(), nullable=True))
+        batch_op.add_column(sa.Column('languages', sa.JSON(), nullable=True))
+
+    with op.batch_alter_table('roms', schema=None) as batch_op:
+        # Set default values for languages and regions
+        batch_op.execute("UPDATE roms SET languages = '[]'")
+        batch_op.execute("UPDATE roms SET regions = JSON_ARRAY(region)")
+        batch_op.drop_column('region')
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('roms', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('region', mysql.VARCHAR(length=20), nullable=True))
+
+    with op.batch_alter_table('roms', schema=None) as batch_op:
+        batch_op.execute("UPDATE roms SET region = JSON_UNQUOTE(JSON_EXTRACT(regions, '$[0]'))")
+        batch_op.drop_column('languages')
+        batch_op.drop_column('regions')
+
+    # ### end Alembic commands ###

backend/endpoints/rom.py

@@ -66,13 +66,14 @@ class RomSchema(BaseModel):
     has_cover: bool
     url_cover: str
 
-    region: Optional[str]
     revision: Optional[str]
-    tags: list
+    regions: list[str]
+    languages: list[str]
+    tags: list[str]
     multi: bool
-    files: list
-    url_screenshots: list
-    path_screenshots: list
+    files: list[str]
+    url_screenshots: list[str]
+    path_screenshots: list[str]
     full_path: str
     download_path: str
 

backend/models/rom.py

@@ -56,8 +56,9 @@ class Rom(BaseModel):
     path_cover_l: str = Column(Text, default=DEFAULT_PATH_COVER_L)
     url_cover: str = Column(Text, default=DEFAULT_PATH_COVER_L)
 
-    region: str = Column(String(20))
     revision: str = Column(String(20))
+    regions: JSON = Column(JSON, default=[])
+    languages: JSON = Column(JSON, default=[])
     tags: JSON = Column(JSON, default=[])
     multi: bool = Column(Boolean, default=False)
     files: JSON = Column(JSON, default=[])

backend/utils/__init__.py

@@ -1,28 +1,25 @@
 import re
-
+import numpy as np
 
 LANGUAGES = [
-    "Ar",
-    "Da",
-    "De",
-    "En",
-    "En-US",
-    "Es",
-    "Fi",
-    "Fr",
-    "It",
-    "Ja",
-    "Ko",
-    "Nl",
-    "Pl",
-    "Pt",
-    "Pt-BR",
-    "Ru",
-    "Sv",
-    "Zh",
-    "Zh-Hans",
-    "Zh-Hant",
-    "nolang",
+    ("Ar", "Arabic"),
+    ("Da", "Danish"),
+    ("De", "German"),
+    ("En", "English"),
+    ("Es", "Spanish"),
+    ("Fi", "Finnish"),
+    ("Fr", "French"),
+    ("It", "Italian"),
+    ("Ja", "Japanese"),
+    ("Ko", "Korean"),
+    ("Nl", "Dutch"),
+    ("No", "Norwegian"),
+    ("Pl", "Polish"),
+    ("Pt", "Portuguese"),
+    ("Ru", "Russian"),
+    ("Sv", "Swedish"),
+    ("Zh", "Chinese"),
+    ("nolang", "No Language"),
 ]
 
 REGIONS = [
@@ -58,31 +55,43 @@
 REGIONS_BY_SHORTCODE = {region[0].lower(): region[1] for region in REGIONS}
 REGIONS_NAME_KEYS = [region[1].lower() for region in REGIONS]
 
+LANGUAGES_BY_SHORTCODE = {lang[0].lower(): lang[1] for lang in LANGUAGES}
+LANGUAGES_NAME_KEYS = [lang[1].lower() for lang in LANGUAGES]
+
 TAG_REGEX = r"\(([^)]+)\)|\[([^]]+)\]"
 EXTENSION_REGEX = r"\.(([a-z]+\.)*\w+)$"
 
 
 def parse_tags(file_name: str) -> tuple:
-    reg = ""
     rev = ""
+    regs = []
+    langs = []
     other_tags = []
-    tags = re.findall(TAG_REGEX, file_name)
-
-    for p_tag, s_tag in tags:
-        tag = p_tag or s_tag
+    tags = [tag[0] or tag[1] for tag in re.findall(TAG_REGEX, file_name)]
+    tags = np.array([tag.split(",") for tag in tags]).flatten()
+    tags = [tag.strip() for tag in tags]
 
+    for tag in tags:
         if tag.lower() in REGIONS_BY_SHORTCODE.keys():
-            reg = REGIONS_BY_SHORTCODE[tag.lower()]
+            regs.append(REGIONS_BY_SHORTCODE[tag.lower()])
             continue
 
         if tag.lower() in REGIONS_NAME_KEYS:
-            reg = tag
+            regs.append(tag)
+            continue
+
+        if tag.lower() in LANGUAGES_BY_SHORTCODE.keys():
+            langs.append(LANGUAGES_BY_SHORTCODE[tag.lower()])
+            continue
+
+        if tag.lower() in LANGUAGES_NAME_KEYS:
+            langs.append(tag)
             continue
 
         if "reg" in tag.lower():
             match = re.match(r"^reg[\s|-](.*)$", tag, re.IGNORECASE)
             if match:
-                reg = (
+                regs.append(
                     REGIONS_BY_SHORTCODE[match.group(1).lower()]
                     if match.group(1).lower() in REGIONS_BY_SHORTCODE.keys()
                     else match.group(1)
@@ -96,7 +105,7 @@ def parse_tags(file_name: str) -> tuple:
                 continue
 
         other_tags.append(tag)
-    return reg, rev, other_tags
+    return regs, rev, langs, other_tags
 
 
 def get_file_name_with_no_tags(file_name: str) -> str:

backend/utils/fastapi.py

@@ -62,7 +62,7 @@ async def scan_rom(
         multi_files=rom_attrs["files"],
         roms_path=roms_path,
     )
-    reg, rev, other_tags = parse_tags(rom_attrs["file_name"])
+    regs, rev, langs, other_tags = parse_tags(rom_attrs["file_name"])
     rom_attrs.update(
         {
             "file_path": roms_path,
@@ -72,8 +72,9 @@ async def scan_rom(
             "file_size": file_size,
             "file_size_units": file_size_units,
             "multi": rom_attrs["multi"],
-            "region": reg,
+            "regions": regs,
             "revision": rev,
+            "languages": langs,
             "tags": other_tags,
         }
     )

backend/utils/tests/test_utils.py

@@ -7,25 +7,28 @@
 
 def test_parse_tags():
     file_name = "Super Mario Bros. (World).nes"
-    assert parse_tags(file_name) == ("World", "", [])
+    assert parse_tags(file_name) == (["World"], "", [], [])
 
     file_name = "Super Mario Bros. (W) (Rev A).nes"
-    assert parse_tags(file_name) == ("World", "A", [])
+    assert parse_tags(file_name) == (["World"], "A", [], [])
 
     file_name = "Super Mario Bros. (USA) (Rev A) (Beta).nes"
-    assert parse_tags(file_name) == ("USA", "A", ["Beta"])
+    assert parse_tags(file_name) == (["USA"], "A", [], ["Beta"])
 
     file_name = "Super Mario Bros. (U) (Beta).nes"
-    assert parse_tags(file_name) == ("USA", "", ["Beta"])
+    assert parse_tags(file_name) == (["USA"], "", [], ["Beta"])
 
     file_name = "Super Mario Bros. (CH) [!].nes"
-    assert parse_tags(file_name) == ("China", "", ["!"])
+    assert parse_tags(file_name) == (["China"], "", [], ["!"])
 
     file_name = "Super Mario Bros. (reg-T) (rev-1.2).nes"
-    assert parse_tags(file_name) == ("Taiwan", "1.2", [])
+    assert parse_tags(file_name) == (["Taiwan"], "1.2", [], [])
 
     file_name = "Super Mario Bros. (Reg S) (Rev A).nes"
-    assert parse_tags(file_name) == ("Spain", "A", [])
+    assert parse_tags(file_name) == (["Spain"], "A", [], [])
+
+    file_name = "Super Metroid (Japan, USA) (En,Ja).zip"
+    assert parse_tags(file_name) == (["Japan", "USA"], "", ["English", "Japanese"], [])
 
 
 def test_get_file_name_with_no_tags():

frontend/src/components/Details/Info.vue

@@ -10,23 +10,23 @@ const downloadStore = storeDownload();
     class="d-flex align-center text-body-1 mt-0 py-2"
     no-gutters
   >
-    <v-col cols="2" xs="2" sm="2" md="2" lg="1" xl="1" class="font-weight-medium"
-      ><span>File</span></v-col
-    >
-    <v-col class="text-body-1"
-      ><span>{{ rom.file_name }}</span></v-col
-    >
+    <v-col cols="2" xs="2" sm="2" md="2" lg="1" xl="1" class="font-weight-medium">
+      <span>File</span>
+    </v-col>
+    <v-col class="text-body-1">
+      <span>{{ rom.file_name }}</span>
+    </v-col>
   </v-row>
   <v-row
     v-if="rom.multi"
     class="d-flex align-center text-body-1 mt-0 py-2"
     no-gutters
   >
-    <v-col cols="2" xs="2" sm="2" md="2" lg="1" xl="1" class="font-weight-medium"
-      ><span>Files</span></v-col
-    >
-    <v-col
-      ><v-select
+    <v-col cols="2" xs="2" sm="2" md="2" lg="1" xl="1" class="font-weight-medium">
+      <span>Files</span>
+    </v-col>
+    <v-col>
+      <v-select
         :label="rom.file_name"
         item-title="file_name"
         v-model="downloadStore.filesToDownloadMultiFileRom"
@@ -39,49 +39,51 @@ const downloadStore = storeDownload();
         hide-details
         clearable
         chips
-    /></v-col>
+      />
+    </v-col>
   </v-row>
   <v-row class="d-flex align-center text-body-1 mt-0 py-2" no-gutters>
-    <v-col cols="2" xs="2" sm="2" md="2" lg="1" xl="1" class="font-weight-medium"
-      ><span>Size</span></v-col
-    >
-    <v-col
-      ><span>{{ rom.file_size }} {{ rom.file_size_units }}</span></v-col
-    >
+    <v-col cols="2" xs="2" sm="2" md="2" lg="1" xl="1" class="font-weight-medium">
+      <span>Size</span>
+    </v-col>
+    <v-col>
+      <span>{{ rom.file_size }} {{ rom.file_size_units }}</span>
+    </v-col>
   </v-row>
   <v-row
-    v-if="rom.r_igdb_id"
+    v-if="rom.igdb_id"
     class="d-flex align-center text-body-1 py-2"
     no-gutters
   >
-    <v-col cols="2" xs="2" sm="2" md="2" lg="1" xl="1" class="font-weight-medium"
-      ><span>IGDB</span></v-col
-    >
+    <v-col cols="2" xs="2" sm="2" md="2" lg="1" xl="1" class="font-weight-medium">
+      <span>IGDB</span>
+    </v-col>
     <v-col>
       <v-chip
         variant="outlined"
         class="text-romm-accent-1"
         :href="`https://www.igdb.com/games/${rom.slug}`"
         label
-        >{{ rom.r_igdb_id }}</v-chip
       >
+        {{ rom.igdb_id }}
+      </v-chip>
     </v-col>
   </v-row>
   <v-row
     v-if="rom.tags.length > 0"
     class="d-flex align-center text-body-1 mt-0 py-2"
     no-gutters
   >
-    <v-col cols="2" xs="2" sm="2" md="2" lg="1" xl="1" class="font-weight-medium"
-      ><span>Tags</span></v-col
-    >
-    <v-col
-      ><v-chip-group class="pt-0"
-        ><v-chip v-for="tag in rom.tags" :key="tag" class="bg-chip" label>{{
-          tag
-        }}</v-chip></v-chip-group
-      ></v-col
-    >
+    <v-col cols="2" xs="2" sm="2" md="2" lg="1" xl="1" class="font-weight-medium">
+      <span>Tags</span>
+    </v-col>
+    <v-col>
+      <v-chip-group class="pt-0">
+        <v-chip v-for="tag in rom.tags" :key="tag" class="bg-chip" label>
+          {{ tag }}
+        </v-chip>
+      </v-chip-group>
+    </v-col>
   </v-row>
   <v-row class="d-flex py-3" no-gutters>
     <v-col class="font-weight-medium text-caption">