Update asset build script and new glyphs

.gitignore

@@ -62,3 +62,10 @@ dkms.conf
 
 # Built files
 build/
+
+# FEH asset files
+asset/apk/*
+asset/collection/*
+asset/file/apk/*
+asset/file/collection/*
+!**/list.txt

asset/local_configs.json

@@ -1,22 +1,18 @@
 {
     "FEH": {
         "linux": "/mnt/c/Users/laqie/Projects/FEH/",
-        "darwin": "/Users/laqieer/Projects/FEH/"
+        "darwin": "/Users/laqieer/Projects/FEH/",
+        "win32": "asset/file/apk/"
     },
     "CKB2WAV": {
         "linux": "asset/tool/CKBtoWav.exe",
-        "darwin": "asset/tool/cktool extract"
-    },
-    "Sound": {
-        "linux": [
-            "assets/TWZH/Sound/",
-            "assets/JPJA/Sound/"
-        ],
-        "darwin": [
-            "assets/TWZH/Sound/",
-            "assets/JPJA/Sound/"
-        ]
+        "darwin": "asset/tool/cktool extract",
+        "win32": "asset\\tool\\CKBtoWav.exe"
     },
+    "Sound": [
+        "assets/TWZH/Sound/",
+        "assets/JPJA/Sound/"
+    ],
     "voice": {
         "freq": 10512
     }

asset/script/common.py

@@ -8,13 +8,20 @@
 
 local_configs = {}
 
-with open("asset/local_configs.json", 'r') as f:
+with open("asset/local_configs.json", 'r', encoding='utf-8') as f:
     local_configs = json.load(f)
 
 for k, v in local_configs.items():
     if sys.platform in v:
         local_configs[k] = v[sys.platform]
 
+def index_files_in_path(path):
+    file_paths = {}
+    for root, dirs, files in os.walk(path):
+        for filename in files:
+            file_paths[filename] = root
+    return file_paths
+
 def find_file_in_path(filename, path, recursive=True):
     if not recursive:
         if os.path.isfile(os.path.join(path, filename)):
@@ -26,7 +33,7 @@ def find_file_in_path(filename, path, recursive=True):
     return None
 
 def parse_background_color_in_grit(grit_path):
-    with open(grit_path, 'r') as f:
+    with open(grit_path, 'r', encoding='utf-8') as f:
         for line in f:
             result = re.findall(r'-gT([0-9a-fA-F]+)', line)
             if result:

asset/script/count_units_in_map.py

@@ -11,7 +11,7 @@
     for file in files:
         if re.match(r'S\d{4}C\.json$', file):
             path = os.path.join(root, file)
-            with open(path, 'r') as f:
+            with open(path, 'r', encoding='utf-8') as f:
                 text = f.read()
                 units = re.findall(r'"id_tag": "([^"]+)",', text)
                 if units:

asset/script/make_backgrounds.py

@@ -12,27 +12,29 @@
 
 backgrounds = set()
 
+file_paths = common.index_files_in_path('asset/file/collection/Backgrounds')
+
 def load_backgrounds_in_scenarios(folder):
     for root, dirs, files in os.walk(folder):
         for file in files:
             if re.match(r'S\d{4}\.json$', file):
                 path = os.path.join(root, file)
-                with open(path, 'r') as f:
+                with open(path, 'r', encoding='utf-8') as f:
                     scenarios = json.load(f)
                     for scenario in scenarios:
                         if scenario['key'] in ('MID_SCENARIO_OPENING_IMAGE', 'MID_SCENARIO_ENDING_IMAGE'):
                             backgrounds.add(scenario['value'])
 
 def make_backgrounds(src_folder, dst_folder, filename):
-    with open(filename, 'w') as f:
+    with open(filename, 'w', encoding='utf-8') as f:
         f.write('#include "background.h"\n')
         f.write('#include "backgrounds.h"\n')
         f.write('#include "gfx_background.h"\n')
         f.write('\n')
         f.write('struct BackgroundEntNew const newBackgrounds[] = {\n')
         for background in sorted(backgrounds):
             background_file = background + '.png'
-            src = os.path.join(src_folder, background_file)
+            src = os.path.join(file_paths.get(background_file, src_folder), background_file)
             if not os.path.exists(src):
                 warnings.warn('%s does not exist' % background_file)
                 continue
@@ -61,7 +63,7 @@ def make_backgrounds(src_folder, dst_folder, filename):
         f.write('};\n')
 
 def make_header(filename):
-    with open(filename, 'w') as f:
+    with open(filename, 'w', encoding='utf-8') as f:
         f.write('#pragma once\n')
         f.write('\n')
         f.write('#define BACKGROUND_NEW 21\n')

asset/script/make_faces.py

@@ -10,19 +10,23 @@
 import warnings
 from PIL import Image
 
+# doc on faces: https://feheroes.fandom.com/wiki/Module:ScenarioArchiveToWiki/data
+
 faces = []
 hero_faces = {}
 face_counts = {}
 face_ids = {}
 
+file_paths = common.index_files_in_path('asset/file/collection/Character Art + Sprites')
+
 def is_generic_face(face):
     return face.startswith('ch90_')
 
 def load_hero_faces(folder):
     for root, dirs, files in os.walk(folder):
         for file in files:
             path = os.path.join(root, file)
-            with open(path, 'r') as f:
+            with open(path, 'r', encoding='utf-8') as f:
                 heros = json.load(f)
                 for hero in heros:
                     if hero['face_name'] and not is_generic_face(hero['face_name']):
@@ -33,7 +37,7 @@ def load_faces_in_scenarios(folder):
         for file in files:
             if re.match(r'S\d{4}\.json$', file):
                 path = os.path.join(root, file)
-                with open(path, 'r') as f:
+                with open(path, 'r', encoding='utf-8') as f:
                     scenarios = json.load(f)
                     for scenario in scenarios:
                         if scenario['key'] in ('MID_SCENARIO_OPENING', 'MID_SCENARIO_ENDING', 'MID_SCENARIO_MAP_BEGIN', 'MID_SCENARIO_MAP_END'):
@@ -49,7 +53,7 @@ def load_heroes_in_maps(folder):
         for file in files:
             if re.match(r'S\d{4}C\.json$', file):
                 path = os.path.join(root, file)
-                with open(path, 'r') as f:
+                with open(path, 'r', encoding='utf-8') as f:
                     data = json.load(f)
                     for unit in data['units']:
                         if unit['id_tag'] in hero_faces:
@@ -79,7 +83,7 @@ def clear_faces(folder):
 
 def make_faces(src_folder, dst_folder, filename):
     clear_faces(dst_folder)
-    with open(filename, 'w') as f:
+    with open(filename, 'w', encoding='utf-8') as f:
         f.write('#include "face.h"\n')
         f.write('#include "faceNew.h"\n')
         f.write('#include "facesNew.h"\n')
@@ -89,24 +93,25 @@ def make_faces(src_folder, dst_folder, filename):
         f.write('\n')
         f.write('const struct FaceInfoNew newFaces[] = {\n')
         for face in faces:
-            if not os.path.exists(os.path.join(src_folder, face)):
+            file_path = file_paths.get(face + '.ssbp', os.path.join(src_folder, face))
+            if not os.path.exists(file_path):
                 warnings.warn('%s does not exist' % face)
                 continue
             face_file = 'Face.png'
-            src = os.path.join(src_folder, face, face_file)
+            src = os.path.join(file_path, face_file)
             im = None
             try:
                 im = Image.open(src)
             except:
-                for root, dirs, files in os.walk(os.path.join(src_folder, face)):
+                for root, dirs, files in os.walk(file_path):
                     for file in files:
                         if re.match(r'Face_Normal\w+\.png$', file):
                             face_file = file
                             src = os.path.join(root, file)
                             im = Image.open(src)
                             break
             if im is None:
-                for root, dirs, files in os.walk(os.path.join(src_folder, face)):
+                for root, dirs, files in os.walk(file_path):
                     for file in files:
                         if re.match(r'Face_Cool\w+\.png$', file):
                             face_file = file
@@ -126,7 +131,7 @@ def make_faces(src_folder, dst_folder, filename):
             im.save(dst)
             hasChibiFace = True
             try:
-                src = os.path.join(src_folder, face, 'Face_FC.png')
+                src = os.path.join(file_path, 'Face_FC.png')
                 dst = os.path.join(dst_folder, 'chibi', '%s_Face_FC.png' % face)
                 im = Image.open(src)
                 im2 = Image.new('RGBA', (33, 32))
@@ -164,7 +169,7 @@ def make_faces(src_folder, dst_folder, filename):
         f.write('};\n')
 
 def make_header(filename):
-    with open(filename, 'w') as f:
+    with open(filename, 'w', encoding='utf-8') as f:
         f.write('#pragma once\n')
         f.write('\n')
         f.write('enum {\n')
@@ -181,7 +186,7 @@ def make_header(filename):
             f.write('#define %s "\\x%x\\x%x"\n' % (face, id & 0xff, id >> 8))
 
 def make_test_text(filename):
-    with open(filename, 'w') as f:
+    with open(filename, 'w', encoding='utf-8') as f:
         f.write('#include "textNew.h"\n')
         f.write('#include "facesNew.h"\n')
         f.write('\n')

asset/script/make_heroes.py

@@ -12,7 +12,7 @@
 hero_data = {}
 
 def load_hero_ids(header_file):
-    with open(header_file, 'r') as f:
+    with open(header_file, 'r', encoding='utf-8') as f:
         lines = f.readlines()
         for line in lines:
             result = re.findall(r'([PE]ID_\w+)', line)
@@ -23,14 +23,14 @@ def load_hero_data(folder):
     for root, dirs, files in os.walk(folder):
         for file in files:
             path = os.path.join(root, file)
-            with open(path, 'r') as f:
+            with open(path, 'r', encoding='utf-8') as f:
                 heros = json.load(f)
                 for hero in heros:
                     if hero['id_tag'] in hero_ids:
                         hero_data[hero['id_tag']] = hero
 
 def make_heroes(filename):
-    with open(filename, 'w') as f:
+    with open(filename, 'w', encoding='utf-8') as f:
         f.write('#include "heroes.h"\n')
         f.write('#include "hero_jobs.h"\n')
         f.write('#include "unit.h"\n')
@@ -73,7 +73,7 @@ def make_heroes(filename):
         f.write('};\n')
 
 def make_debug_heroes(filename):
-    with open(filename, 'w') as f:
+    with open(filename, 'w', encoding='utf-8') as f:
         f.write('#include "heroes.h"\n')
         f.write('#include "hero_jobs.h"\n')
         f.write('#include "debugchapter.h"\n')

asset/script/make_texts.py

@@ -7,6 +7,7 @@
 import json
 import opencc
 import warnings
+import unicodedata
 from enum import Enum
 
 class Language(Enum):
@@ -44,7 +45,7 @@ class TextType(Enum):
 converter = opencc.OpenCC('t2s.json')
 
 def load_hero_ids(header_file):
-    with open(header_file, 'r') as f:
+    with open(header_file, 'r', encoding='utf-8') as f:
         lines = f.readlines()
         for line in lines:
             result = re.findall(r'([PE]ID_\w+) (\d+)', line)
@@ -55,7 +56,7 @@ def read_unit_infos(folder):
     for root, dirs, files in os.walk(folder):
         for file in files:
             path = os.path.join(root, file)
-            with open(path, 'r') as f:
+            with open(path, 'r', encoding='utf-8') as f:
                 units = json.load(f)
                 for unit in units:
                     unit_infos[unit['id_tag']] = unit
@@ -73,7 +74,7 @@ def count_units_and_skills(folder):
         for file in files:
             if re.match(r'S\d{4}C\.json$', file):
                 path = os.path.join(root, file)
-                with open(path, 'r') as f:
+                with open(path, 'r', encoding='utf-8') as f:
                     units =json.load(f)['units']
                     for unit in units:
                         if unit['id_tag'] not in unit_counts:
@@ -112,6 +113,7 @@ def show_unit_face(matched):
 def convert_text(language, text):
     if language == Language.LANGUAGE_CHINESE:
         text = converter.convert(text) # convert to simplified Chinese
+    text = unicodedata.normalize('NFKC', text)
     text = text.replace('\\"', '"') # unescape double quotes
     text = text.replace('"', '\\"') # escape double quotes
     text = '"' + text + '"' # wrap with double quotes
@@ -164,7 +166,7 @@ def read_texts(type, language, folder):
         for file in files:
             if should_read_file(type, file):
                 path = os.path.join(root, file)
-                with open(path, 'r') as f:
+                with open(path, 'r', encoding='utf-8') as f:
                     kvs = json.load(f)
                     for kv in kvs:
                         key = kv['key']
@@ -232,7 +234,7 @@ def validate_text_key(key):
     return key.replace('、', '').replace('＋', 'P').replace('―', '_')
 
 def write_texts(type, filename):
-    with open(filename, 'w') as f:
+    with open(filename, 'w', encoding='utf-8') as f:
         f.write('// This file is generated by make_texts.py\n')
         f.write('// Text type: %s\n' % type)
         for key in sorted(get_text_keys(type)):
@@ -249,7 +251,7 @@ def write_texts(type, filename):
             f.write('},\n')
 
 def write_header(filename):
-    with open(filename, 'w') as f:
+    with open(filename, 'w', encoding='utf-8') as f:
         f.write('// This file is generated by make_texts.py\n')
         f.write('\n')
         f.write('#pragma once\n')
@@ -262,7 +264,7 @@ def write_header(filename):
             text_id += 1
 
 def write_names(filename):
-    with open(filename, 'w') as f:
+    with open(filename, 'w', encoding='utf-8') as f:
         f.write('#include "texts.h"\n')
         f.write('\n')
         f.write('const char * const text_keys[] = {\n')