diff --git a/UnityFileSystem.Tests/UnityFileSystemTests.cs b/UnityFileSystem.Tests/UnityFileSystemTests.cs index 7001a91..394c26d 100644 --- a/UnityFileSystem.Tests/UnityFileSystemTests.cs +++ b/UnityFileSystem.Tests/UnityFileSystemTests.cs @@ -11,6 +11,27 @@ namespace UnityDataTools.FileSystem.Tests; #pragma warning disable NUnit2005, NUnit2006 +// Copies a file into a fresh temp directory whose name and file name contain non-ASCII +// (Japanese) characters, runs the test against it, then removes the directory. +internal static class NonAsciiPath +{ + public static void WithCopy(string source, string fileName, Action test) + { + var dir = Path.Combine(Path.GetTempPath(), "UnityDataTools_テスト_" + Path.GetRandomFileName()); + Directory.CreateDirectory(dir); + try + { + var path = Path.Combine(dir, fileName); + File.Copy(source, path); + test(path); + } + finally + { + Directory.Delete(dir, true); + } + } +} + public class ArchiveTests : AssetBundleTestFixture { public ArchiveTests(Context context) : base(context) @@ -62,6 +83,21 @@ public void MountArchive_ValidArchive_ReturnsArchive() archive.Dispose(); } + // Paths are marshalled to the native library as UTF-8, so non-ASCII paths must work. + [Test] + public void MountArchive_NonAsciiPath_ReturnsArchive() + { + var source = Path.Combine(Context.UnityDataFolder, "assetbundle"); + + NonAsciiPath.WithCopy(source, "アセット.bundle", path => + { + UnityArchive archive = null; + Assert.DoesNotThrow(() => archive = UnityFileSystem.MountArchive(path, "archive:/")); + Assert.IsNotNull(archive); + archive.Dispose(); + }); + } + [Ignore("This test doesn't return the expected error, this condition is probably not handled correctly in Unity")] public void DisposeArchive_ValidArchive_UnmountsArchive() { @@ -155,6 +191,24 @@ public void OpenFile_LocalFileSystem_ReturnsValidFile() Assert.DoesNotThrow(() => file.Dispose()); } + // Paths are marshalled to the native library as UTF-8, so a file at a non-ASCII path + // must open and read correctly (this failed when paths were marshalled as ANSI). + [Test] + public void OpenFile_NonAsciiPath_ReadsExpectedData() + { + var source = Path.Combine(Context.TestDataFolder, "TextFile.txt"); + + NonAsciiPath.WithCopy(source, "テキスト.txt", path => + { + using var file = UnityFileSystem.OpenFile(path); + var buffer = new Byte[1000]; + var actualSize = file.Read(1000, buffer); + + Assert.AreEqual(21, actualSize); + Assert.AreEqual("This is my text file.", Encoding.UTF8.GetString(buffer, 0, (int)actualSize)); + }); + } + [Test] public void GetFileSize_LocalFileSystem_ReturnSize() { diff --git a/UnityFileSystem/DllWrapper.cs b/UnityFileSystem/DllWrapper.cs index 2649ade..dacc8a7 100644 --- a/UnityFileSystem/DllWrapper.cs +++ b/UnityFileSystem/DllWrapper.cs @@ -183,7 +183,7 @@ public static class DllWrapper [DllImport("UnityFileSystemApi", CallingConvention = CallingConvention.Cdecl, EntryPoint = "UFS_MountArchive")] - public static extern ReturnCode MountArchive([MarshalAs(UnmanagedType.LPStr)] string path, [MarshalAs(UnmanagedType.LPStr)] string mountPoint, out UnityArchiveHandle handle); + public static extern ReturnCode MountArchive([MarshalAs(UnmanagedType.LPUTF8Str)] string path, [MarshalAs(UnmanagedType.LPUTF8Str)] string mountPoint, out UnityArchiveHandle handle); [DllImport("UnityFileSystemApi", CallingConvention = CallingConvention.Cdecl, @@ -195,6 +195,10 @@ public static class DllWrapper EntryPoint = "UFS_GetArchiveNodeCount")] public static extern ReturnCode GetArchiveNodeCount(UnityArchiveHandle handle, out int count); + // Strings returned from native (here and in the other StringBuilder-based calls) come back as + // UTF-8 bytes but are marshalled as the system ANSI code page, so only ASCII round-trips correctly. + // These are internal archive/serialized-file names, which Unity keeps ASCII, so it's not an issue in + // practice. Input paths, by contrast, are marshalled as UTF-8 (LPUTF8Str) to support non-ASCII paths. [DllImport("UnityFileSystemApi", CallingConvention = CallingConvention.Cdecl, EntryPoint = "UFS_GetArchiveNode")] @@ -203,14 +207,38 @@ public static class DllWrapper [DllImport("UnityFileSystemApi", CallingConvention = CallingConvention.Cdecl, EntryPoint = "UFS_CreateArchive")] - public static extern ReturnCode CreateArchive([MarshalAs(UnmanagedType.LPArray, ArraySubType = UnmanagedType.LPStr)] string[] sourceFiles, - [MarshalAs(UnmanagedType.LPArray, ArraySubType = UnmanagedType.LPStr)] string[] aliases, bool[] isSerializedFile, int count, - [MarshalAs(UnmanagedType.LPStr)] string archiveFile, CompressionType compression, out int crc); + private static extern ReturnCode CreateArchiveNative(IntPtr[] sourceFiles, IntPtr[] aliases, bool[] isSerializedFile, int count, + [MarshalAs(UnmanagedType.LPUTF8Str)] string archiveFile, CompressionType compression, out int crc); + + // The native library expects UTF-8 paths. LPUTF8Str handles the scalar strings, but it + // can't be used as an array subtype, so the string arrays are marshalled to UTF-8 by hand. + public static ReturnCode CreateArchive(string[] sourceFiles, string[] aliases, bool[] isSerializedFile, int count, + string archiveFile, CompressionType compression, out int crc) + { + var sourcePtrs = new IntPtr[sourceFiles.Length]; + var aliasPtrs = new IntPtr[aliases.Length]; + try + { + for (int i = 0; i < sourceFiles.Length; ++i) + sourcePtrs[i] = Marshal.StringToCoTaskMemUTF8(sourceFiles[i]); + for (int i = 0; i < aliases.Length; ++i) + aliasPtrs[i] = Marshal.StringToCoTaskMemUTF8(aliases[i]); + + return CreateArchiveNative(sourcePtrs, aliasPtrs, isSerializedFile, count, archiveFile, compression, out crc); + } + finally + { + foreach (var p in sourcePtrs) + Marshal.FreeCoTaskMem(p); + foreach (var p in aliasPtrs) + Marshal.FreeCoTaskMem(p); + } + } [DllImport("UnityFileSystemApi", CallingConvention = CallingConvention.Cdecl, EntryPoint = "UFS_OpenFile")] - public static extern ReturnCode OpenFile([MarshalAs(UnmanagedType.LPStr)] string path, out UnityFileHandle handle); + public static extern ReturnCode OpenFile([MarshalAs(UnmanagedType.LPUTF8Str)] string path, out UnityFileHandle handle); [DllImport("UnityFileSystemApi", CallingConvention = CallingConvention.Cdecl, EntryPoint = "UFS_ReadFile")] @@ -235,7 +263,7 @@ public static extern ReturnCode ReadFile(UnityFileHandle handle, long size, [DllImport("UnityFileSystemApi", CallingConvention = CallingConvention.Cdecl, EntryPoint = "UFS_OpenSerializedFile")] - public static extern ReturnCode OpenSerializedFile([MarshalAs(UnmanagedType.LPStr)] string path, out SerializedFileHandle handle); + public static extern ReturnCode OpenSerializedFile([MarshalAs(UnmanagedType.LPUTF8Str)] string path, out SerializedFileHandle handle); [DllImport("UnityFileSystemApi", CallingConvention = CallingConvention.Cdecl, @@ -270,13 +298,13 @@ public static extern ReturnCode ReadFile(UnityFileHandle handle, long size, [DllImport("UnityFileSystemApi", CallingConvention = CallingConvention.Cdecl, EntryPoint = "UFS_GetRefTypeTypeTree")] - public static extern ReturnCode GetRefTypeTypeTree(SerializedFileHandle handle, [MarshalAs(UnmanagedType.LPStr)] string className, - [MarshalAs(UnmanagedType.LPStr)] string namespaceName, [MarshalAs(UnmanagedType.LPStr)] string assemblyName, out TypeTreeHandle typeTree); + public static extern ReturnCode GetRefTypeTypeTree(SerializedFileHandle handle, [MarshalAs(UnmanagedType.LPUTF8Str)] string className, + [MarshalAs(UnmanagedType.LPUTF8Str)] string namespaceName, [MarshalAs(UnmanagedType.LPUTF8Str)] string assemblyName, out TypeTreeHandle typeTree); [DllImport("UnityFileSystemApi", CallingConvention = CallingConvention.Cdecl, EntryPoint = "UFS_AddTypeTreeSourceFromFile")] - public static extern ReturnCode AddTypeTreeSourceFromFile([MarshalAs(UnmanagedType.LPStr)] string path, out long handle); + public static extern ReturnCode AddTypeTreeSourceFromFile([MarshalAs(UnmanagedType.LPUTF8Str)] string path, out long handle); [DllImport("UnityFileSystemApi", CallingConvention = CallingConvention.Cdecl,