Script Tools Update, Support Emulating Last Write Time, Error Handling Improvements, Other Stuff

.gitmodules

@@ -3,5 +3,4 @@
 	url = https://github.com/Heroes-Hacking-Central/Heroes.SDK.git
 [submodule "Submodules/Atlus-Script-Tools"]
 	path = Submodules/Atlus-Script-Tools
-	url = https://github.com/AnimatedSwine37/Atlus-Script-Tools.git
-	branch = bf-emulator
+	url = https://github.com/tge-was-taken/Atlus-Script-Tools.git

Emulator/BF.File.Emulator/Bf/BfBuilder.cs

@@ -105,10 +105,38 @@ public void AddEnumFile(string filePath)
         }
     }
 
+    /// <summary>
+    /// Tries to get all files that the base flow imports
+    /// </summary>
+    /// <param name="flowFormat">The format of the flowscript</param>
+    /// <param name="library">The library to use for the flowscript</param>
+    /// <param name="encoding">The encoding of the flowscript</param>
+    /// <param name="foundImports">An array of absolute paths to all files that the base flows import,
+    /// both directly and transitively. This includes the base files.</param>
+    /// <returns></returns>
+    public bool TryGetImports(FlowFormatVersion flowFormat, Library library, Encoding encoding,
+        out string[] foundImports)
+    {
+        // Use compiler arg overrides (if they're there)
+        if (_library != null) library = _library;
+        if (_encoding != null) encoding = _encoding;
+        if (_flowFormat != null) flowFormat = (FlowFormatVersion)_flowFormat;
+
+        var compiler = new FlowScriptCompiler(flowFormat);
+        compiler.Library = OverrideLibraries(library);
+        compiler.Encoding = encoding;
+
+        var imports = new List<string>();
+        imports.AddRange(_flowFiles);
+        imports.AddRange(_msgFiles);
+
+        return compiler.TryGetImports(imports, out foundImports);
+    }
+
     /// <summary>
     /// Builds a BF file.
     /// </summary>
-    public unsafe Stream? Build(IntPtr originalHandle, string originalPath, FlowFormatVersion flowFormat, Library library, Encoding encoding, AtlusLogListener? listener = null, bool noBaseBf = false)
+    public EmulatedBf? Build(IntPtr originalHandle, string originalPath, FlowFormatVersion flowFormat, Library library, Encoding encoding, AtlusLogListener? listener = null, bool noBaseBf = false)
     {
         _log?.Info("[BfEmulator] Building BF File | {0}", originalPath);
 
@@ -135,21 +163,43 @@ public void AddEnumFile(string filePath)
             imports.AddRange(_flowFiles.GetRange(1, _flowFiles.Count - 1));
         imports.AddRange(_msgFiles);
 
-        if (!compiler.TryCompileWithImports(bfStream, imports, baseFlow, out FlowScript flowScript))
+        try
         {
-            _log?.Error("[BfEmulator] Failed to compile BF File | {0}", originalPath);
-            return null;
-        }
+            if (!compiler.TryCompileWithImports(bfStream, imports, baseFlow, out FlowScript flowScript,
+                    out var sources))
+            {
+                _log?.Error("[BfEmulator] Failed to compile BF File | {0}", originalPath);
+                return null;
+            }
 
-        // Return the compiled bf
-        var bfBinary = flowScript.ToBinary();
-        var stream = StreamUtils.CreateMemoryStream(bfBinary.Header.FileSize);
-        bfBinary.ToStream(stream, true);
-        stream.Position = 0;
+            // Return the compiled bf
+            var bfBinary = flowScript.ToBinary();
+            var stream = StreamUtils.CreateMemoryStream(bfBinary.Header.FileSize);
+            bfBinary.ToStream(stream, true);
+            stream.Position = 0;
 
-        return stream;
+            DateTime lastWrite = sources.Where(x => x != null).Select(Fiel.GetLastWriteTimeUtc).Max();
+            return new EmulatedBf(stream, sources, lastWrite);
+        }
+        catch (Exception exception)
+        {
+            var flows = string.Join(", ", _flowFiles.Concat(_msgFiles));
+            _log.Error(
+                "[BF Builder] Failed to compile bf {0} with source files: {1}. This may be due to your mods not being translated. Error: {2}",
+                originalPath, flows, exception.Message);
+            if (exception.StackTrace != null)
+                _log.Error(exception.StackTrace);
+            return null;
+        }
     }
 
+    /// <summary>
+    /// Applies library file overrides to the base library.
+    /// This adds aliases to functions with different names, replaces functions with different
+    /// return values or parameters, and adds any completely new functions.
+    /// </summary>
+    /// <param name="library">The base library to override</param>
+    /// <returns>A deep copy of the base library with overrides applied</returns>
     private Library OverrideLibraries(Library library)
     {
         if (_libraryEnums.Count == 0 && _libraryFuncs.Count == 0) return library;
@@ -176,7 +226,15 @@ private Library OverrideLibraries(Library library)
 
             if (module != -1 && index != -1)
             {
-                library.FlowScriptModules[module].Functions[index] = func;
+                var existingFunc = library.FlowScriptModules[module].Functions[index];
+                if (FlowFunctionsSame(existingFunc, func))
+                {
+                    existingFunc.Aliases.Add(func.Name);
+                }
+                else
+                {
+                    library.FlowScriptModules[module].Functions[index] = func;
+                }
             }
             else
             {
@@ -188,4 +246,17 @@ private Library OverrideLibraries(Library library)
         library.FlowScriptModules[0].Enums.AddRange(_libraryEnums);
         return library;
     }
-}
+
+    /// <summary>
+    /// Checks if two flowscript functions are effectively the same.
+    /// They are the same if the return type and parameter types are the same.
+    /// </summary>
+    /// <param name="func1">The first function to compare</param>
+    /// <param name="func2">The other function to compare</param>
+    /// <returns>Truee if the two functions are effectively the same, false otherwise</returns>
+    private bool FlowFunctionsSame(FlowScriptModuleFunction func1, FlowScriptModuleFunction func2)
+    {
+        return func1.ReturnType == func2.ReturnType && func1.Parameters.Select(param => param.Type)
+            .SequenceEqual(func2.Parameters.Select(param => param.Type));
+    }
+}

Emulator/BF.File.Emulator/Bf/BfBuilderFactory.cs

@@ -148,7 +148,7 @@ private void OverrideCompilerArgs(string file)
 
         _flowFormat = GetFlowScriptFormatVersion(outFormat);
         _library = LibraryLookup.GetLibrary(args.Library);
-        _encoding = AtlusEncoding.GetByName(args.Encoding);
+        _encoding = AtlusEncoding.Create(args.Encoding);
 
         _log.Info($"[BfBuilderFactory] Changed script compiler args to OutFormat: {args.OutFormat}, Library: {args.Library}, Encoding: {args.Encoding}");
     }

Emulator/BF.File.Emulator/Bf/EmulatedBf.cs

@@ -0,0 +1,31 @@
+namespace BF.File.Emulator.Bf;
+
+/// <summary>
+/// Contains information about a BF file that has been emulated
+/// </summary>
+public class EmulatedBf
+{
+    /// <summary>
+    /// A list of full paths to all source files used to compile this bf.
+    /// This includes all msg, bf, and flow files that are imported excluding the base bf.
+    /// </summary>
+    public List<string> Sources { get; }
+
+    /// <summary>
+    /// The stream for the emulated file
+    /// </summary>
+    public Stream Stream { get; }
+
+    /// <summary>
+    /// The last write time of the file
+    /// This is set as the maximum last write time of all sources when the file was first emulated
+    /// </summary>
+    public DateTime LastWriteTime { get; }
+
+    public EmulatedBf(Stream stream, List<String> sources, DateTime lastWriteTime)
+    {
+        Sources = sources;
+        Stream = stream;
+        LastWriteTime = lastWriteTime;
+    }
+}

Emulator/BF.File.Emulator/BfEmulator.cs

@@ -25,7 +25,7 @@ public class BfEmulator : IEmulator
 
     // Note: Handle->Stream exists because hashing IntPtr is easier; thus can resolve reads faster.
     private readonly BfBuilderFactory _builderFactory;
-    private readonly ConcurrentDictionary<string, Stream?> _pathToStream = new(StringComparer.OrdinalIgnoreCase);
+    private readonly ConcurrentDictionary<string, EmulatedBf?> _pathToEmulated = new(StringComparer.OrdinalIgnoreCase);
     private Logger _log;
 
     private FlowFormatVersion _flowFormat;
@@ -46,17 +46,17 @@ public BfEmulator(Logger log, bool dumpFiles, Game game)
             case Game.P3P:
                 _flowFormat = FlowFormatVersion.Version1;
                 _library = LibraryLookup.GetLibrary("P3P");
-                _encoding = AtlusEncoding.GetByName("P4");
+                _encoding = AtlusEncoding.Create("P3P_EFIGS");
                 break;
             case Game.P4G:
                 _flowFormat = FlowFormatVersion.Version1;
                 _library = LibraryLookup.GetLibrary("P4G");
-                _encoding = AtlusEncoding.GetByName("P4");
+                _encoding = AtlusEncoding.Create("P4G_EFIGS");
                 break;
             case Game.P5R:
                 _flowFormat = FlowFormatVersion.Version3BigEndian;
                 _library = LibraryLookup.GetLibrary("P5R");
-                _encoding = AtlusEncoding.GetByName("P5R");
+                _encoding = AtlusEncoding.Create("P5R_EFIGS");
                 break;
         }
     }
@@ -65,13 +65,13 @@ public bool TryCreateFile(IntPtr handle, string filepath, string route, out IEmu
     {
         // Check if we already made a custom BF for this file.
         emulated = null!;
-        if (_pathToStream.TryGetValue(filepath, out var stream))
+        if (_pathToEmulated.TryGetValue(filepath, out var emulatedBf))
         {
             // Avoid recursion into same file.
-            if (stream == null)
+            if (emulatedBf == null)
                 return false;
 
-            emulated = new EmulatedFile<Stream>(stream);
+            emulated = new EmulatedFile<Stream>(emulatedBf.Stream, emulatedBf.LastWriteTime);
             return true;
         }
 
@@ -109,15 +109,16 @@ public bool TryCreateEmulatedFile(IntPtr handle, string srcDataPath, string outp
             return false;
 
         // Make the BF file.
-        _pathToStream[outputPath] = null; // Avoid recursion into same file.
+        _pathToEmulated[outputPath] = null; // Avoid recursion into same file.
 
-        stream = builder!.Build(handle, srcDataPath, _flowFormat, _library, _encoding, _listener, isEmpty);
-        if (stream == null)
+        var emulatedBf = builder!.Build(handle, srcDataPath, _flowFormat, _library, _encoding, _listener, isEmpty);
+        if (emulatedBf == null)
             return false;
 
-        _pathToStream.TryAdd(outputPath, stream);
-        emulated = new EmulatedFile<Stream>(stream);
-        _log.Info("[BfEmulator] Created Emulated file with Path {0}", outputPath);
+        stream = emulatedBf.Stream;
+        _pathToEmulated.TryAdd(outputPath, emulatedBf);
+        emulated = new EmulatedFile<Stream>(stream, emulatedBf.LastWriteTime);
+        _log.Info("[BfEmulator] Created Emulated file with Path {0} and Last Write {1}", outputPath, emulatedBf.LastWriteTime);
 
         if (DumpFiles)
             DumpFile(route, stream);
@@ -143,13 +144,13 @@ public void OnModLoading(string modFolder)
     /// <param name="bfPath">Full path to the file.</param>
     public void UnregisterFile(string bfPath)
     {
-        _pathToStream!.Remove(bfPath, out var stream);
-        stream?.Dispose();
+        _pathToEmulated!.Remove(bfPath, out var emulated);
+        emulated?.Stream.Dispose();
     }
 
-    public void RegisterFile(string destinationPath, Stream stream)
+    public void RegisterFile(string destinationPath, Stream stream, DateTime lastWriteTime)
     {
-        _pathToStream.TryAdd(destinationPath, stream);
+        _pathToEmulated.TryAdd(destinationPath, new EmulatedBf(stream, new List<string>(), lastWriteTime));
     }
 
     private void DumpFile(string route, Stream stream)
@@ -162,11 +163,25 @@ private void DumpFile(string route, Stream stream)
         _log.Info($"[BfEmulator] Written To {dumpPath}");
     }
 
+    internal bool TryGetImports(string route, out string[] imports)
+    {
+        if (!_builderFactory.TryCreateFromPath(route, out var builder))
+        {
+            imports = Array.Empty<string>();
+            return false;
+        }
+
+        return builder!.TryGetImports(_flowFormat, _library, _encoding, out imports);
+    }
+
     internal List<RouteGroupTuple> GetInput() => _builderFactory.RouteFileTuples;
 
     internal void AddFromFolders(string dir) => _builderFactory.AddFromFolders(dir);
 
     internal void AddFile(string file, string route) => _builderFactory.AddFile(Path.GetFileName(file), file, Path.GetDirectoryName(file)!, route);
+
+    internal void SetEncoding(string encoding) => _encoding = AtlusEncoding.Create(encoding);
+
 }
 
 public class AtlusLogListener : LogListener

Emulator/BF.File.Emulator/BfEmulatorApi.cs

@@ -1,4 +1,5 @@
-using BF.File.Emulator.Interfaces;
+using System.Diagnostics;
+using BF.File.Emulator.Interfaces;
 using BF.File.Emulator.Interfaces.Structures.IO;
 using BF.File.Emulator.Utilities;
 using FileEmulationFramework.Interfaces;
@@ -7,6 +8,9 @@
 using Microsoft.Win32.SafeHandles;
 using System.Runtime.InteropServices;
 
+// Aliasing for readability, since our assembly name has priority over 'File'
+using Fiel = System.IO.File;
+
 namespace BF.File.Emulator;
 
 public class BfEmulatorApi : IBfEmulator
@@ -57,8 +61,9 @@ public void RegisterBf(string sourcePath, string destinationPath)
 
         Native.SetFilePointerEx(handle, 0, IntPtr.Zero, 0);
         var fileStream = new FileStream(new SafeFileHandle(handle, true), FileAccess.Read);
-        var emulated = new EmulatedFile<FileStream>(fileStream);
-        _bfEmulator.RegisterFile(destinationPath, fileStream);
+        var lastWrite = Fiel.GetLastWriteTimeUtc(sourcePath);
+        var emulated = new EmulatedFile<FileStream>(fileStream, lastWrite);
+        _bfEmulator.RegisterFile(destinationPath, fileStream, lastWrite);
         _framework.RegisterVirtualFile(destinationPath, emulated, false);
 
         _logger.Info("[BfEmulatorApi] Registered bf {0} at {1}", sourcePath, destinationPath);
@@ -97,4 +102,14 @@ public void AddDirectory(string dir)
         _bfEmulator.AddFromFolders(dir);
     }
 
+    public void SetEncoding(string encoding)
+    {
+        _logger.Info("Setting encoding to {0}", encoding);
+        _bfEmulator.SetEncoding(encoding);
+    }
+
+    public bool TryGetImports(string route, out string[] imports)
+    {
+        return _bfEmulator.TryGetImports(route, out imports);
+    }
 }