refactor: update how affixes are parsed

Aside from some black formatting and debug message fixes in this commit
the main change is around parsing affixes.

The parser now looks for the affix tables and then indexes which affix
files to actually parse.

This now includes the Tinkerer and prepares the work for the unique
affixes that only occur on some items like the Black Pearl.

Fixes #70, #60
Refers #24, #51

Pipfile

@@ -14,3 +14,6 @@ numpy = "~=1.22.4"
 
 [dev-packages]
 black = "*"
+
+[pipenv]
+allow_prereleases = true

run.py

@@ -11,17 +11,10 @@
 from tqdb.utils.text import texts
 
 # Disable any DEBUG logging from PIL:
-logging.getLogger('PIL').setLevel(logging.WARNING)
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s %(levelname)s %(message)s',
-    datefmt='%H:%M:%S')
+logging.getLogger("PIL").setLevel(logging.WARNING)
 
 # Supported languages
-LANGUAGES = [
-    'cs', 'de', 'en', 'es', 'fr', 'it',
-    'ja', 'ko', 'pl', 'ru', 'uk', 'zh'
-]
+LANGUAGES = ["cs", "de", "en", "es", "fr", "it", "ja", "ko", "pl", "ru", "uk", "zh"]
 
 
 def tqdb_language(language):
@@ -30,22 +23,22 @@ def tqdb_language(language):
 
     """
     # Prepare the texts based on the language:
-    logging.info(f'Parsing locale: {language}')
+    logging.info(f"Parsing locale: {language}")
     texts.load_locale(language)
 
     data = {
-        'affixes': main.parse_affixes(),
-        'creatures': main.parse_creatures(),
-        'equipment': main.parse_equipment(),
-        'quests': main.parse_quests(),
-        'sets': main.parse_sets(),
-        'skills': main.parse_skills(),
+        "affixes": main.parse_affixes(),
+        "creatures": main.parse_creatures(),
+        "equipment": main.parse_equipment(),
+        "quests": main.parse_quests(),
+        "sets": main.parse_sets(),
+        "skills": main.parse_skills(),
     }
 
-    logging.info('Writing output to files...')
+    logging.info("Writing output to files...")
 
-    output_name = paths.OUTPUT / f'tqdb.{language.lower()}.{tqdb_version}.json'
-    with open(output_name, 'w', encoding='utf8') as data_file:
+    output_name = paths.OUTPUT / f"tqdb.{language.lower()}.{tqdb_version}.json"
+    with open(output_name, "w", encoding="utf8") as data_file:
         json.dump(data, data_file, ensure_ascii=False, sort_keys=True)
 
 
@@ -57,74 +50,74 @@ def tqdb_parse():
     which will allow faster parsing into the individual languages.
 
     """
-    logging.info('Parsing database into its intermediate format…')
+    logging.info("Parsing database into its intermediate format…")
 
     if not os.path.exists(paths.CACHE):
         os.makedirs(paths.CACHE)
 
     data = {
-        'affixes': main.parse_affixes,
-        'creatures': main.parse_creatures,
-        'equipment': main.parse_equipment,
-        'quests': main.parse_quests,
-        'sets': main.parse_sets,
-        'skills': main.parse_skills,
+        "affixes": main.parse_affixes,
+        "creatures": main.parse_creatures,
+        "equipment": main.parse_equipment,
+        "quests": main.parse_quests,
+        "sets": main.parse_sets,
+        "skills": main.parse_skills,
     }
 
     for category, fn in data.items():
-        logging.info(f'Parsing {data}')
+        logging.info(f"Parsing {data}")
         data = fn()
-        output_name = paths.CACHE / f'tqdb.{category}.json'
-        with open(output_name, 'w', encoding='utf8') as data_file:
+        output_name = paths.CACHE / f"tqdb.{category}.json"
+        with open(output_name, "w", encoding="utf8") as data_file:
             json.dump(data, data_file, ensure_ascii=False, sort_keys=True)
 
 
-
 def tqdb():
     """
     Run the Titan Quest Database parser.
 
     """
     # Parse command line parameters
-    argparser = argparse.ArgumentParser(
-        description='TQ Database parser',
-        formatter_class=argparse.RawTextHelpFormatter)
+    argparser = argparse.ArgumentParser(description="TQ Database parser", formatter_class=argparse.RawTextHelpFormatter)
     argparser.add_argument(
-        '-l',
-        '--locale',
-        action='store',
-        default='en',
+        "-l",
+        "--locale",
+        action="store",
+        default="en",
         type=lambda s: s.lower(),
-        choices=['cs', 'de', 'en', 'es', 'fr', 'it', 'ja', 'ko', 'pl',
-                 'ru', 'uk', 'zh'],
-        help=('Specify the two letter locale you want to parse (default: en)\n'
-              'cs - Czech\n'
-              'de - German\n'
-              'en - English\n'
-              'es - Spanish\n'
-              'fr - French\n'
-              'it - Italian\n'
-              'ja - Japanese\n'
-              'ko - Korean\n'
-              'pl - Polish\n'
-              'ru - Russian\n'
-              'uk - Ukrainian\n'
-              'zh - Chinese\n'))
-    argparser.add_argument(
-        '-f',
-        '--force',
-        action='store',
-        default=False,
-        dest='force_parsing')
+        choices=["cs", "de", "en", "es", "fr", "it", "ja", "ko", "pl", "ru", "uk", "zh"],
+        help=(
+            "Specify the two letter locale you want to parse (default: en)\n"
+            "cs - Czech\n"
+            "de - German\n"
+            "en - English\n"
+            "es - Spanish\n"
+            "fr - French\n"
+            "it - Italian\n"
+            "ja - Japanese\n"
+            "ko - Korean\n"
+            "pl - Polish\n"
+            "ru - Russian\n"
+            "uk - Ukrainian\n"
+            "zh - Chinese\n"
+        ),
+    )
+    argparser.add_argument("-f", "--force", action="store", default=False, dest="force_parsing")
+    argparser.add_argument("-a", "--all-languages", action="store_true", default=False, dest="all_languages")
+
     argparser.add_argument(
-        '-a',
-        '--all-languages',
-        action='store_true',
-        default=False,
-        dest='all_languages')
+        "-d",
+        "--debug",
+        help="Print debug statements from parser",
+        action="store_const",
+        dest="loglevel",
+        const=logging.DEBUG,
+        default=logging.INFO,
+    )
 
     # Grab the arguments:
     args = argparser.parse_args()
+    logging.basicConfig(level=args.loglevel, format="%(asctime)s %(levelname)s %(message)s", datefmt="%H:%M:%S")
 
     def create_sprite_sheet():
         start_time = time.time()
@@ -158,5 +151,5 @@ def create_sprite_sheet():
     create_sprite_sheet()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     tqdb()

tqdb/__init__.py

@@ -2,4 +2,4 @@
 Modules to parse Titan Quest Equipment, skills, and more!
 
 """
-__version__ = "1.4.1"
+__version__ = "1.4.2"

tqdb/constants/resources.py

@@ -4,11 +4,14 @@
 """
 from pathlib import Path
 
-AFFIXES = [
-    "records/item*/lootmagicalaffixes/*ix/default/*.dbr",
-    "records/xpack*/item*/lootmagicalaffixes/*ix/default/*.dbr",
-    "records/xpack*/item*/lootmagicalaffixes/*ix/*.dbr",
-]
+# This is being replaced by only parsing loot affixes that appear in
+# the `randomizerName` properties of the AFFIX_TABLES.
+# This way we know for sure those affixes can occur
+# AFFIXES = [
+#     "records/item*/lootmagicalaffixes/*ix/default/*.dbr",
+#     "records/xpack*/item*/lootmagicalaffixes/*ix/default/*.dbr",
+#     "records/xpack*/item*/lootmagicalaffixes/*ix/*.dbr",
+# ]
 
 AFFIX_TABLES = [
     "records/item*/lootmagicalaffixes/*ix/tables*/*.dbr",
@@ -189,91 +192,91 @@
         "records/xpack/item/containers/bosschest05_hades_03.dbr",
     ],
     "x2tagMonsterName096": [
-        "records/xpack2/item/containers/boss containers/" "bosschest01_porcus_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest01_porcus_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest01_porcus_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest01_porcus_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest01_porcus_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest01_porcus_03.dbr",
     ],
     "x2tagMonsterName184": [
-        "records/xpack2/item/containers/boss containers/" "bosschest03_mineghost_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest03_mineghost_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest03_mineghost_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest03_mineghost_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest03_mineghost_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest03_mineghost_03.dbr",
     ],
     "x2tagMonsterName147": [
-        "records/xpack2/item/containers/boss containers/" "bosschest05_ancients_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest05_ancients_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest05_ancients_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest05_ancients_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest05_ancients_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest05_ancients_03.dbr",
     ],
     "x2tagMonsterName148": [
-        "records/xpack2/item/containers/boss containers/" "bosschest05_ancients_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest05_ancients_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest05_ancients_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest05_ancients_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest05_ancients_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest05_ancients_03.dbr",
     ],
     "x2tagMonsterName149": [
-        "records/xpack2/item/containers/boss containers/" "bosschest05_ancients_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest05_ancients_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest05_ancients_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest05_ancients_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest05_ancients_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest05_ancients_03.dbr",
     ],
     "x2tagMonsterName150": [
-        "records/xpack2/item/containers/boss containers/" "bosschest05_ancients_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest05_ancients_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest05_ancients_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest05_ancients_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest05_ancients_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest05_ancients_03.dbr",
     ],
     "x2tagMonsterName011": [
-        "records/xpack2/item/containers/boss containers/" "bosschest07_hildisvini_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest07_hildisvini_02.dbr",
-        "records/xpack2/item/containers/boss containers" "/bosschest07_hildisvini_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest07_hildisvini_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest07_hildisvini_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest07_hildisvini_03.dbr",
     ],
     "x2tagMonsterName121": [
-        "records/xpack2/item/containers/boss containers/" "bosschest09_shroomking_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest09_shroomking_02.dbr",
-        "records/xpack2/item/containers/boss containers" "/bosschest09_shroomking_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest09_shroomking_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest09_shroomking_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest09_shroomking_03.dbr",
     ],
     "x2tagasgard_odin": [
-        "records/xpack2/item/containers/boss containers/" "bosschest10_aesir_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest10_aesir_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest10_aesir_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest10_aesir_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest10_aesir_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest10_aesir_03.dbr",
     ],
     "x2tagasgard_freyja": [
-        "records/xpack2/item/containers/boss containers/" "bosschest10_aesir_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest10_aesir_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest10_aesir_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest10_aesir_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest10_aesir_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest10_aesir_03.dbr",
     ],
     "x2tagasgard_baldr": [
-        "records/xpack2/item/containers/boss containers/" "bosschest10_aesir_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest10_aesir_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest10_aesir_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest10_aesir_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest10_aesir_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest10_aesir_03.dbr",
     ],
     "x2tagasgard_tyr": [
-        "records/xpack2/item/containers/boss containers/" "bosschest10_aesir_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest10_aesir_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest10_aesir_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest10_aesir_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest10_aesir_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest10_aesir_03.dbr",
     ],
     "x2tagutgard_mimer": [
-        "records/xpack2/item/containers/boss containers/" "bosschest12_mimir_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest12_mimir_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest12_mimir_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest12_mimir_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest12_mimir_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest12_mimir_03.dbr",
     ],
     "x2tagMonsterName093": [
-        "records/xpack2/item/containers/boss containers/" "bosschest14_loki_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest14_loki_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest14_loki_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest14_loki_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest14_loki_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest14_loki_03.dbr",
     ],
     "x2tagmuspelheim_loki": [
-        "records/xpack2/item/containers/boss containers/" "bosschest14_loki_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest14_loki_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest14_loki_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest14_loki_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest14_loki_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest14_loki_03.dbr",
     ],
     "x2tagMonsterName058": [
-        "records/xpack2/item/containers/boss containers/" "bosschest15_surtr_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest15_surtr_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest15_surtr_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest15_surtr_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest15_surtr_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest15_surtr_03.dbr",
     ],
     "x2tagMonsterName059": [
-        "records/xpack2/item/containers/boss containers/" "bosschest15_surtr_01.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest15_surtr_02.dbr",
-        "records/xpack2/item/containers/boss containers/" "bosschest15_surtr_03.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest15_surtr_01.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest15_surtr_02.dbr",
+        "records/xpack2/item/containers/boss containers/bosschest15_surtr_03.dbr",
     ],
-    "x2tagMonsterName094": ["", "", "records/xpack2/item/containers/boss containers/" "bosschest17_fafnir_03.dbr"],
+    "x2tagMonsterName094": ["", "", "records/xpack2/item/containers/boss containers/bosschest17_fafnir_03.dbr"],
 }
 
 CREATURES = [