TSJ and TSV parsing (#1962)

* Deserialization support for tsv files * Benchmarking * Apparently moving the setter out of the lambda fixed the setAccessible issue * Thread it * Use AllArgsConstructor instead of field reflection * Clean up AllArgsConstructor TSV deserialization * Refactor TsvUtils * Remove AllArgsConstructors from Excels * Set field accessible * [WIP] TSJ improvements * [WIP] More TSV stuff * [WIP] More TSV stuff * Working TSV parser (slow) * Load Excels in TSJ > JSON > TSV priority
Grasscutters · Nov 23, 2022 · 0b53295 · 0b53295
1 parent 46b0c7c
commit 0b53295
Show file tree

Hide file tree

Showing 16 changed files with 754 additions and 65 deletions.
diff --git a/src/main/java/emu/grasscutter/data/ResourceLoader.java b/src/main/java/emu/grasscutter/data/ResourceLoader.java
@@ -11,7 +11,10 @@
 import emu.grasscutter.game.world.SpawnDataEntry.GridBlockId;
 import emu.grasscutter.game.world.SpawnDataEntry.SpawnGroupEntry;
 import emu.grasscutter.scripts.SceneIndexManager;
+import emu.grasscutter.utils.FileUtils;
 import emu.grasscutter.utils.JsonUtils;
+import emu.grasscutter.utils.TsvUtils;
+import it.unimi.dsi.fastutil.Pair;
 import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
 import it.unimi.dsi.fastutil.ints.IntArrayList;
 import it.unimi.dsi.fastutil.ints.IntArraySet;
@@ -23,6 +26,8 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CopyOnWriteArraySet;
 import java.util.regex.Pattern;
 import java.util.stream.Stream;
 
@@ -32,8 +37,9 @@
 
 public class ResourceLoader {
 
-    private static final List<String> loadedResources = new ArrayList<>();
+    private static final Set<String> loadedResources = new CopyOnWriteArraySet<>();
 
+    // Get a list of all resource classes, sorted by loadPriority
     public static List<Class<?>> getResourceDefClasses() {
         Reflections reflections = new Reflections(ResourceLoader.class.getPackage().getName());
         Set<?> classes = reflections.getSubTypesOf(GameResource.class);
@@ -51,6 +57,25 @@ public static List<Class<?>> getResourceDefClasses() {
         return classList;
     }
 
+    // Get a list containing sets of all resource classes, sorted by loadPriority
+    protected static List<Set<Class<?>>> getResourceDefClassesPrioritySets() {
+        val reflections = new Reflections(ResourceLoader.class.getPackage().getName());
+        val classes = reflections.getSubTypesOf(GameResource.class);
+        val priorities = ResourceType.LoadPriority.getInOrder();
+        Grasscutter.getLogger().debug("Priorities are "+priorities);
+        val map = new LinkedHashMap<ResourceType.LoadPriority, Set<Class<?>>>(priorities.size());
+        priorities.forEach(p -> map.put(p, new HashSet<>()));
+
+        classes.forEach(c -> {
+            // val c = (Class<?>) o;
+            val annotation = c.getAnnotation(ResourceType.class);
+            if (annotation != null) {
+                map.get(annotation.loadPriority()).add(c);
+            }
+        });
+        return List.copyOf(map.values());
+    }
+
     private static boolean loadedAll = false;
     public static void loadAll() {
         if (loadedAll) return;
@@ -86,48 +111,66 @@ public static void loadResources() {
     }
 
     public static void loadResources(boolean doReload) {
-        for (Class<?> resourceDefinition : getResourceDefClasses()) {
-            ResourceType type = resourceDefinition.getAnnotation(ResourceType.class);
+        long startTime = System.nanoTime();
+        val errors = new ConcurrentLinkedQueue<Pair<String, Exception>>();  // Logger in a parallel stream will deadlock
 
-            if (type == null) {
-                continue;
-            }
-
-            @SuppressWarnings("rawtypes")
-            Int2ObjectMap map = GameData.getMapByResourceDef(resourceDefinition);
+        getResourceDefClassesPrioritySets().forEach(classes -> {
+            classes.stream()
+                .parallel().unordered()
+                .forEach(c -> {
+                    val type = c.getAnnotation(ResourceType.class);
+                    if (type == null) return;
 
-            if (map == null) {
-                continue;
-            }
+                    val map = GameData.getMapByResourceDef(c);
+                    if (map == null) return;
 
-            try {
-                loadFromResource(resourceDefinition, type, map, doReload);
-            } catch (Exception e) {
-                Grasscutter.getLogger().error("Error loading resource file: " + Arrays.toString(type.name()), e.getLocalizedMessage());
-            }
-        }
+                    try {
+                        loadFromResource(c, type, map, doReload);
+                    } catch (Exception e) {
+                        errors.add(Pair.of(Arrays.toString(type.name()), e));
+                    }
+                });
+        });
+        errors.forEach(pair -> Grasscutter.getLogger().error("Error loading resource file: " + pair.left(), pair.right()));
+        long endTime = System.nanoTime();
+        long ns = (endTime - startTime);  //divide by 1000000 to get milliseconds.
+        Grasscutter.getLogger().debug("Loading resources took "+ns+"ns == "+ns/1000000+"ms");
     }
 
     @SuppressWarnings("rawtypes")
     protected static void loadFromResource(Class<?> c, ResourceType type, Int2ObjectMap map, boolean doReload) throws Exception {
-        if (!loadedResources.contains(c.getSimpleName()) || doReload) {
+        val simpleName = c.getSimpleName();
+        if (doReload || !loadedResources.contains(simpleName)) {
             for (String name : type.name()) {
-                loadFromResource(c, name, map);
+                loadFromResource(c, FileUtils.getExcelPath(name), map);
             }
-            loadedResources.add(c.getSimpleName());
-            Grasscutter.getLogger().debug("Loaded " + map.size() + " " + c.getSimpleName() + "s.");
+            loadedResources.add(simpleName);
         }
     }
 
     @SuppressWarnings({"rawtypes", "unchecked"})
-    protected static <T> void loadFromResource(Class<T> c, String fileName, Int2ObjectMap map) throws Exception {
-        List<T> list = JsonUtils.loadToList(getResourcePath("ExcelBinOutput/" + fileName), c);
+    protected static <T> void loadFromResource(Class<T> c, Path filename, Int2ObjectMap map) throws Exception {
+        val results = switch (FileUtils.getFileExtension(filename)) {
+            case "json" -> JsonUtils.loadToList(filename, c);
+            case "tsj" -> TsvUtils.loadTsjToListSetField(c, filename);
+            case "tsv" -> TsvUtils.loadTsvToListSetField(c, filename);
+            default -> null;
+        };
+        if (results == null) return;
+        results.forEach(o -> {
+            GameResource res = (GameResource) o;
+            res.onLoad();
+            map.put(res.getId(), res);
+        });
+    }
 
-        for (T o : list) {
+    @SuppressWarnings({"rawtypes", "unchecked"})
+    protected static <T> void loadFromResource(Class<T> c, String fileName, Int2ObjectMap map) throws Exception {
+        JsonUtils.loadToList(getResourcePath("ExcelBinOutput/" + fileName), c).forEach(o -> {
             GameResource res = (GameResource) o;
             res.onLoad();
             map.put(res.getId(), res);
-        }
+        });
     }
 
     public class ScenePointConfig {  // Sadly this doesn't work as a local class in loadScenePoints()

diff --git a/src/main/java/emu/grasscutter/data/ResourceType.java b/src/main/java/emu/grasscutter/data/ResourceType.java
@@ -2,6 +2,8 @@
 
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
+import java.util.List;
+import java.util.stream.Stream;
 
 @Retention(RetentionPolicy.RUNTIME) 
 public @interface ResourceType {
@@ -28,5 +30,9 @@ public enum LoadPriority {
 		public int value() {
 			return value;
 		}
+
+        public static List<LoadPriority> getInOrder() {
+            return Stream.of(LoadPriority.values()).sorted((x, y) -> y.value() - x.value()).toList();
+        }
 	}
 }
diff --git a/src/main/java/emu/grasscutter/data/excels/ActivityWatcherData.java b/src/main/java/emu/grasscutter/data/excels/ActivityWatcherData.java
@@ -21,7 +21,7 @@ public class ActivityWatcherData extends GameResource {
 
     @Override
     public void onLoad() {
-        triggerConfig.paramList = triggerConfig.paramList.stream().filter(x -> !x.isBlank()).toList();
+        triggerConfig.paramList = triggerConfig.paramList.stream().filter(x -> (x != null) && !x.isBlank()).toList();
         triggerConfig.watcherTriggerType = WatcherTriggerType.getTypeByName(triggerConfig.triggerType);
     }
 

diff --git a/src/main/java/emu/grasscutter/data/excels/BattlePassMissionData.java b/src/main/java/emu/grasscutter/data/excels/BattlePassMissionData.java
@@ -40,8 +40,11 @@ public boolean isValidRefreshType() {
 
     @Override
     public void onLoad() {
-        if (this.getTriggerConfig() != null && getTriggerConfig().getParamList()[0].length() > 0) {
-            this.mainParams = Arrays.stream(getTriggerConfig().getParamList()[0].split("[:;,]")).map(Integer::parseInt).collect(Collectors.toSet());
+        if (this.getTriggerConfig() != null) {
+            var params = getTriggerConfig().getParamList()[0];
+            if ((params != null) && !params.isEmpty()) {
+                this.mainParams = Arrays.stream(params.split("[:;,]")).map(Integer::parseInt).collect(Collectors.toSet());
+            }
         }
     }
 

diff --git a/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddExp.java b/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddExp.java
@@ -1,19 +1,18 @@
 package emu.grasscutter.game.props.ItemUseAction;
 
 import emu.grasscutter.game.props.ItemUseOp;
-import lombok.Getter;
-
-public class ItemUseAddExp extends ItemUseAction {
-    @Getter private int exp = 0;
 
+public class ItemUseAddExp extends ItemUseInt {
     @Override
     public ItemUseOp getItemUseOp() {
         return ItemUseOp.ITEM_USE_ADD_EXP;
     }
 
     public ItemUseAddExp(String[] useParam) {
-        try {
-            this.exp = Integer.parseInt(useParam[0]);
-        } catch (NumberFormatException ignored) {}
+        super(useParam);
+    }
+
+    public int getExp() {
+        return this.i;
     }
 }
diff --git a/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddItem.java b/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddItem.java
@@ -14,7 +14,7 @@ public ItemUseAddItem(String[] useParam) {
         super(useParam);
         try {
             this.count = Integer.parseInt(useParam[1]);
-        } catch (NumberFormatException ignored) {}
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
     }
 
     @Override

diff --git a/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddReliquaryExp.java b/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddReliquaryExp.java
@@ -1,19 +1,14 @@
 package emu.grasscutter.game.props.ItemUseAction;
 
 import emu.grasscutter.game.props.ItemUseOp;
-import lombok.Getter;
-
-public class ItemUseAddReliquaryExp extends ItemUseAction {
-    @Getter private int exp = 0;
 
+public class ItemUseAddReliquaryExp extends ItemUseAddExp {
     @Override
     public ItemUseOp getItemUseOp() {
         return ItemUseOp.ITEM_USE_ADD_RELIQUARY_EXP;
     }
 
     public ItemUseAddReliquaryExp(String[] useParam) {
-        try {
-            this.exp = Integer.parseInt(useParam[0]);
-        } catch (NumberFormatException ignored) {}
+        super(useParam);
     }
 }
diff --git a/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddServerBuff.java b/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddServerBuff.java
@@ -14,7 +14,7 @@ public ItemUseAddServerBuff(String[] useParam) {
         super(useParam);
         try {
             this.duration = Integer.parseInt(useParam[1]);
-        } catch (NumberFormatException ignored) {}
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
     }
 
     @Override

diff --git a/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddWeaponExp.java b/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseAddWeaponExp.java
@@ -1,19 +1,14 @@
 package emu.grasscutter.game.props.ItemUseAction;
 
 import emu.grasscutter.game.props.ItemUseOp;
-import lombok.Getter;
-
-public class ItemUseAddWeaponExp extends ItemUseAction {
-    @Getter private int exp = 0;
 
+public class ItemUseAddWeaponExp extends ItemUseAddExp {
     @Override
     public ItemUseOp getItemUseOp() {
         return ItemUseOp.ITEM_USE_ADD_WEAPON_EXP;
     }
 
     public ItemUseAddWeaponExp(String[] useParam) {
-        try {
-            this.exp = Integer.parseInt(useParam[0]);
-        } catch (NumberFormatException ignored) {}
+        super(useParam);
     }
 }
diff --git a/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseCombineItem.java b/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseCombineItem.java
@@ -15,10 +15,10 @@ public ItemUseCombineItem(String[] useParam) {
         super(useParam);
         try {
             this.resultId = Integer.parseInt(useParam[1]);
-        } catch (NumberFormatException ignored) {}
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
         try {
             this.resultCount = Integer.parseInt(useParam[2]);
-        } catch (NumberFormatException ignored) {}
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
     }
 
     @Override

diff --git a/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseGainAvatar.java b/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseGainAvatar.java
@@ -19,10 +19,10 @@ public ItemUseGainAvatar(String[] useParam) {
         super(useParam);
         try {
             this.level = Integer.parseInt(useParam[1]);
-        } catch (NumberFormatException ignored) {}
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
         try {
             this.constellation = Integer.parseInt(useParam[2]);
-        } catch (NumberFormatException ignored) {}
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
     }
 
     @Override

diff --git a/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseInt.java b/src/main/java/emu/grasscutter/game/props/ItemUseAction/ItemUseInt.java
@@ -8,6 +8,6 @@ public abstract class ItemUseInt extends ItemUseAction {
     public ItemUseInt(String[] useParam) {
         try {
             this.i = Integer.parseInt(useParam[0]);
-        } catch (NumberFormatException ignored) {}
+        } catch (NumberFormatException | ArrayIndexOutOfBoundsException ignored) {}
     }
 }
diff --git a/src/main/java/emu/grasscutter/utils/FileUtils.java b/src/main/java/emu/grasscutter/utils/FileUtils.java
@@ -1,6 +1,7 @@
 package emu.grasscutter.utils;
 
 import emu.grasscutter.Grasscutter;
+import lombok.val;
 
 import java.io.File;
 import java.io.IOException;
@@ -111,6 +112,24 @@ public static Path getResourcePath(String path) {
         return RESOURCES_PATH.resolve(path);
     }
 
+    public static Path getExcelPath(String filename) {
+        return getTsjJsonTsv(RESOURCES_PATH.resolve("ExcelBinOutput"), filename);
+    }
+
+    // Gets path of a resource.
+    // If multiple formats of it exist, priority is TSJ > JSON > TSV
+    // If none exist, return the TSJ path, in case it wants to create a file
+    public static Path getTsjJsonTsv(Path root, String filename) {
+        val name = getFilenameWithoutExtension(filename);
+        val tsj = root.resolve(name + ".tsj");
+        if (Files.exists(tsj)) return tsj;
+        val json = root.resolve(name + ".json");
+        if (Files.exists(json)) return json;
+        val tsv = root.resolve(name + ".tsv");
+        if (Files.exists(tsv)) return tsv;
+        return tsj;
+    }
+
     public static Path getScriptPath(String path) {
         return SCRIPTS_PATH.resolve(path);
     }
@@ -167,14 +186,19 @@ public static void copyResource(String resourcePath, String destination) {
         }
     }
 
-    @Deprecated  // No current uses of this anyway
-    public static String getFilenameWithoutPath(String fileName) {
-        int i = fileName.lastIndexOf(".");
-        if (i > 0) {
-           return fileName.substring(0, i);
-        } else {
-           return fileName;
-        }
+    @Deprecated  // Misnamed legacy function
+    public static String getFilenameWithoutPath(String filename) {
+        return getFilenameWithoutExtension(filename);
+    }
+    public static String getFilenameWithoutExtension(String filename) {
+        int i = filename.lastIndexOf(".");
+        return (i < 0) ? filename : filename.substring(0, i);
+    }
+
+    public static String getFileExtension(Path path) {
+        val filename = path.toString();
+        int i = filename.lastIndexOf(".");
+        return (i < 0) ? "" : filename.substring(i+1);
     }
 
     public static List<Path> getPathsFromResource(String folder) throws URISyntaxException {