extract localization files

Abstractions/ExtractedData.cs

@@ -1,6 +1,7 @@
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using PalworldDataExtractor.Abstractions.Breeding;
+using PalworldDataExtractor.Abstractions.L10N;
 using PalworldDataExtractor.Abstractions.Pals;
 using PalworldDataExtractor.Abstractions.Steam;
 
@@ -14,6 +15,7 @@ public class ExtractedData
     public required IReadOnlyCollection<PalTribe> Tribes { get; init; }
     public required IReadOnlyDictionary<string, byte[]> TribeIcons { get; init; }
     public required IReadOnlyCollection<PalBreedingCombination> UniqueBreedingCombinations { get; init; }
+    public required IReadOnlyDictionary<string, LocalizationFile> LocalizationFiles { get; init; }
 
     public static async Task Serialize(ExtractedData data, Stream outStream) =>
         await JsonSerializer.SerializeAsync(outStream, data, typeof(ExtractedData), ExtractedDataJsonSerializerContext.Default);

Abstractions/L10N/LocalizationFile.cs

@@ -0,0 +1,13 @@
+namespace PalworldDataExtractor.Abstractions.L10N;
+
+public class LocalizationFile
+{
+    public required string Language { get; set; }
+    public IReadOnlyDictionary<string, LocalizationNamespace> Namespaces { get; set; } = new Dictionary<string, LocalizationNamespace>();
+}
+
+public class LocalizationNamespace
+{
+    public required string Namespace { get; set; }
+    public IReadOnlyDictionary<string, string> Fields { get; set; } = new Dictionary<string, string>();
+}

Lib/DataExtractor.cs

@@ -4,6 +4,7 @@
 using CUE4Parse.UE4.VirtualFileSystem;
 using PalworldDataExtractor.Abstractions;
 using PalworldDataExtractor.Abstractions.Breeding;
+using PalworldDataExtractor.Abstractions.L10N;
 using PalworldDataExtractor.Abstractions.Pals;
 using PalworldDataExtractor.Abstractions.Steam;
 using PalworldDataExtractor.Extractors;
@@ -26,15 +27,19 @@ public DataExtractor(string pakFileDirectory, Action<DataExtractorConfiguration>
 
     public async Task<ExtractedData> Extract()
     {
-        SteamManifest? steamManifest = await ExtractSteamManifest();
-        PalTribe[] tribes = await ExtractTribes();
-        IReadOnlyDictionary<string, byte[]> palIcons = await ExtractPalIcons();
-        PalBreedingCombination[] uniqueBreedingCombinations = await ExtractUniqueBreedingCombinations();
+        SteamManifest? steamManifest = await ExtractSteamManifestAsync();
+        PalTribe[] tribes = await ExtractTribesAsync();
+        IReadOnlyDictionary<string, byte[]> palIcons = await ExtractPalIconsAsync();
+        PalBreedingCombination[] uniqueBreedingCombinations = await ExtractUniqueBreedingCombinationsAsync();
+        Dictionary<string, LocalizationFile> localizationFiles = await ExtractLocalizationFilesAsync();
 
-        return new ExtractedData { SteamManifest = steamManifest, Tribes = tribes, TribeIcons = palIcons, UniqueBreedingCombinations = uniqueBreedingCombinations };
+        return new ExtractedData
+        {
+            SteamManifest = steamManifest, Tribes = tribes, TribeIcons = palIcons, UniqueBreedingCombinations = uniqueBreedingCombinations, LocalizationFiles = localizationFiles
+        };
     }
 
-    async Task<SteamManifest?> ExtractSteamManifest()
+    async Task<SteamManifest?> ExtractSteamManifestAsync()
     {
         string absoluteDir = Path.GetFullPath(_pakFileDirectory);
         int indexOfSteamApps = absoluteDir.IndexOf("steamapps", StringComparison.InvariantCultureIgnoreCase);
@@ -49,22 +54,24 @@ public async Task<ExtractedData> Extract()
         return await extractor.Extract();
     }
 
-    async Task<PalTribe[]> ExtractTribes()
+    async Task<PalTribe[]> ExtractTribesAsync()
     {
         IEnumerable<Pal> pals = await new PalsExtractor(_provider).ExtractPalsAsync();
         PalTribe[] tribes = pals.GroupBy(p => p.TribeName).Select(g => new PalTribe { Name = g.Key ?? "???", Pals = g.ToArray() }).ToArray();
         return tribes;
     }
 
-    async Task<IReadOnlyDictionary<string, byte[]>> ExtractPalIcons()
+    async Task<IReadOnlyDictionary<string, byte[]>> ExtractPalIconsAsync()
     {
         IReadOnlyDictionary<string, byte[]> palIcons = await new PalIconsExtractor(_provider).ExtractIconsAsync();
         return palIcons;
     }
 
-    async Task<PalBreedingCombination[]> ExtractUniqueBreedingCombinations() =>
+    async Task<PalBreedingCombination[]> ExtractUniqueBreedingCombinationsAsync() =>
         (await new UniquePalBreedingCombinationExtractor(_provider).ExtractUniquePalBreedingCombinationsAsync()).ToArray();
 
+    async Task<Dictionary<string, LocalizationFile>> ExtractLocalizationFilesAsync() => await new LocalizationFilesExtractor(_provider).ExtractLocalizationFilesAsync();
+
     public void Dispose()
     {
         _provider.Dispose();

Lib/Extractors/LocalizationFilesExtractor.cs

@@ -0,0 +1,68 @@
+using CUE4Parse.FileProvider;
+using CUE4Parse.UE4.Assets.Exports.Engine;
+using CUE4Parse.Utils;
+using PalworldDataExtractor.Abstractions.L10N;
+using PalworldDataExtractor.Readers;
+
+namespace PalworldDataExtractor.Extractors;
+
+public class LocalizationFilesExtractor
+{
+    readonly DefaultFileProvider _provider;
+    readonly UDataTableReader _tableReader;
+
+    public LocalizationFilesExtractor(DefaultFileProvider provider)
+    {
+        _provider = provider;
+        _tableReader = new UDataTableReader(provider);
+    }
+
+    public async Task<Dictionary<string, LocalizationFile>> ExtractLocalizationFilesAsync()
+    {
+        const string l10NFilesRoot = "Pal/Content/L10N/";
+        string[] l10NFiles = _provider.Files.Where(kv => kv.Key.StartsWith(l10NFilesRoot) && kv.Key.EndsWith(".uasset"))
+            .Select(kv => kv.Key.SubstringAfter(l10NFilesRoot).SubstringBeforeLast(".uasset"))
+            .ToArray();
+        IEnumerable<IGrouping<string, string>> l10NFilesByRegion = l10NFiles.GroupBy(f => f.SubstringBefore("/"));
+
+        Dictionary<string, LocalizationFile> dictionary = new();
+        foreach (IGrouping<string, string> group in l10NFilesByRegion)
+        {
+            LocalizationFile file = await ExtractLocalizationFileAsync(group.Key, group.Select(p => l10NFilesRoot + p).ToArray());
+            dictionary.Add(group.Key, file);
+        }
+
+        return dictionary;
+    }
+
+    async Task<LocalizationFile> ExtractLocalizationFileAsync(string language, IEnumerable<string> namespaceFilePaths)
+    {
+        Dictionary<string, LocalizationNamespace> namespaces = new();
+
+        foreach (string path in namespaceFilePaths)
+        {
+            LocalizationNamespace ns = await ExtractLocalizationNamespace(path);
+            namespaces.Add(ns.Namespace, ns);
+        }
+
+        return new LocalizationFile { Language = language, Namespaces = namespaces };
+    }
+
+    async Task<LocalizationNamespace> ExtractLocalizationNamespace(string path)
+    {
+        UDataTable table = await _tableReader.ExtractAsync(path);
+
+        return new LocalizationNamespace
+        {
+            Namespace = table.Name,
+            Fields = table.RowMap.ToDictionary(
+                kv => kv.Key.Text,
+                kv =>
+                {
+                    FStructReader reader = new(kv.Value);
+                    return reader.ParseString("TextData") ?? kv.Key.Text;
+                }
+            )
+        };
+    }
+}

Lib/Extractors/PalsExtractor.cs

@@ -18,7 +18,7 @@ public PalsExtractor(DefaultFileProvider provider)
 
     public async Task<IEnumerable<Pal>> ExtractPalsAsync()
     {
-        UDataTable palTable = await _tableReader.Extract(@"Pal\Content\Pal\DataTable\Character\DT_PalMonsterParameter");
+        UDataTable palTable = await _tableReader.ExtractAsync(@"Pal\Content\Pal\DataTable\Character\DT_PalMonsterParameter");
         return palTable.RowMap.Select((kv, index) => TryParse(index, kv.Key.Text, kv.Value, out Pal? pal) ? pal : null).Where(p => p != null).Select(p => p!);
     }
 

Lib/Extractors/UniquePalBreedingCombinationExtractor.cs

@@ -19,7 +19,7 @@ public UniquePalBreedingCombinationExtractor(DefaultFileProvider provider)
 
     public async Task<IEnumerable<PalBreedingCombination>> ExtractUniquePalBreedingCombinationsAsync()
     {
-        UDataTable palTable = await _tableReader.Extract(@"Pal\Content\Pal\DataTable\Character\DT_PalCombiUnique");
+        UDataTable palTable = await _tableReader.ExtractAsync(@"Pal\Content\Pal\DataTable\Character\DT_PalCombiUnique");
 
         List<PalBreedingCombination> result = new();
 

Lib/Readers/FStructReader.cs

@@ -1,4 +1,5 @@
 using CUE4Parse.UE4.Assets.Objects;
+using CUE4Parse.UE4.Objects.Core.i18N;
 using CUE4Parse.UE4.Objects.UObject;
 
 namespace PalworldDataExtractor.Readers;
@@ -14,16 +15,16 @@ public FStructReader(FStructFallback obj)
 
     public string? ParseEnumValue(string property, string prefix)
     {
-        string? valueString = ParseString(property);
-        if (valueString == null || !valueString.StartsWith(prefix))
+        string? valueString = ParseString(property) ?? _obj.GetOrDefault<FName>(property).Text;
+        if (valueString == "" || valueString == "None" || !valueString.StartsWith(prefix))
         {
             return null;
         }
 
         return valueString[prefix.Length..];
     }
 
-    public string? ParseString(string property) => (string?)_obj.GetOrDefault<string>(property) ?? _obj.GetOrDefault<FName>(property).Text;
+    public string? ParseString(string property) => (string?)_obj.GetOrDefault<string>(property) ?? ((FText?)_obj.GetOrDefault<FText>(property))?.Text;
     public int ParseInt(string property) => _obj.GetOrDefault<int>(property);
     public float ParseFloat(string property) => _obj.GetOrDefault<float>(property);
     public bool ParseBool(string property) => _obj.GetOrDefault<bool>(property);

Lib/Readers/UDataTableReader.cs

@@ -13,7 +13,7 @@ public UDataTableReader(DefaultFileProvider provider)
         _provider = provider;
     }
 
-    public async Task<UDataTable> Extract(string file)
+    public async Task<UDataTable> ExtractAsync(string file)
     {
         List<UObject> objects = (await _provider.LoadAllObjectsAsync(file)).ToList();
         if (objects.Count == 0)

Tests/TestFiles/extracted_data.expected

@@ -75,5 +75,45 @@
       "ParentTribeB": "tribe2",
       "ChildCharacterId": "child"
     }
-  ]
+  ],
+  "LocalizationFiles": {
+    "language1": {
+      "Language": "language1",
+      "Namespaces": {
+        "namespace1": {
+          "Namespace": "namespace1",
+          "Fields": {
+            "field1": "value1 language1",
+            "field2": "value2 language1"
+          }
+        },
+        "namespace2": {
+          "Namespace": "namespace2",
+          "Fields": {
+            "field3": "value3 language1",
+            "field4": "value4 language1"
+          }
+        }
+      }
+    },
+    "language2": {
+      "Language": "language2",
+      "Namespaces": {
+        "namespace1": {
+          "Namespace": "namespace1",
+          "Fields": {
+            "field1": "value1 language2",
+            "field2": "value2 language2"
+          }
+        },
+        "namespace2": {
+          "Namespace": "namespace2",
+          "Fields": {
+            "field3": "value3 language2",
+            "field4": "value4 language2"
+          }
+        }
+      }
+    }
+  }
 }

Tests/UnitTests/ExtractedDataTest.cs

@@ -2,6 +2,7 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using PalworldDataExtractor.Abstractions;
 using PalworldDataExtractor.Abstractions.Breeding;
+using PalworldDataExtractor.Abstractions.L10N;
 using PalworldDataExtractor.Abstractions.Pals;
 using PalworldDataExtractor.Abstractions.Steam;
 using Tests.Tools;
@@ -96,6 +97,77 @@ public class ExtractedDataTest
             UniqueBreedingCombinations = new[]
             {
                 new PalBreedingCombination("tribe1", "tribe2", "child")
+            },
+            LocalizationFiles = new Dictionary<string, LocalizationFile>
+            {
+                {
+                    "language1",
+                    new LocalizationFile
+                    {
+                        Language = "language1",
+                        Namespaces = new Dictionary<string, LocalizationNamespace>
+                        {
+                            {
+                                "namespace1",
+                                new LocalizationNamespace
+                                {
+                                    Namespace = "namespace1",
+                                    Fields = new Dictionary<string, string>
+                                    {
+                                        { "field1", "value1 language1" },
+                                        { "field2", "value2 language1" }
+                                    }
+                                }
+                            },
+                            {
+                                "namespace2",
+                                new LocalizationNamespace
+                                {
+                                    Namespace = "namespace2",
+                                    Fields = new Dictionary<string, string>
+                                    {
+                                        { "field3", "value3 language1" },
+                                        { "field4", "value4 language1" }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                },
+                {
+                    "language2",
+                    new LocalizationFile
+                    {
+                        Language = "language2",
+                        Namespaces = new Dictionary<string, LocalizationNamespace>
+                        {
+                            {
+                                "namespace1",
+                                new LocalizationNamespace
+                                {
+                                    Namespace = "namespace1",
+                                    Fields = new Dictionary<string, string>
+                                    {
+                                        { "field1", "value1 language2" },
+                                        { "field2", "value2 language2" }
+                                    }
+                                }
+                            },
+                            {
+                                "namespace2",
+                                new LocalizationNamespace
+                                {
+                                    Namespace = "namespace2",
+                                    Fields = new Dictionary<string, string>
+                                    {
+                                        { "field3", "value3 language2" },
+                                        { "field4", "value4 language2" }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
             }
         };