diff --git a/Test/UnrealFlagsTests.cs b/Test/UnrealFlagsTests.cs index 459c640..341c721 100644 --- a/Test/UnrealFlagsTests.cs +++ b/Test/UnrealFlagsTests.cs @@ -1,5 +1,6 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using UELib.Branch; +using UELib.Core; using UELib.Flags; namespace Eliot.UELib.Test @@ -33,5 +34,15 @@ public void TestUnrealPackageFlags() Assert.IsTrue(flags.HasFlags((uint)DefaultEngineBranch.PackageFlagsDefault.AllowDownload)); Assert.IsFalse(flags.HasFlags((uint)DefaultEngineBranch.PackageFlagsDefault.ClientOptional)); } + + [TestMethod] + public void TestBulkDataToCompressionFlags() + { + const BulkDataFlags dataFlags = BulkDataFlags.Unused | BulkDataFlags.CompressedLZX; + + var compressionFlags = dataFlags.ToCompressionFlags(); + Assert.IsTrue(compressionFlags.HasFlag(CompressionFlags.ZLX)); + Assert.IsTrue(compressionFlags == CompressionFlags.ZLX); + } } -} \ No newline at end of file +} diff --git a/Test/UnrealStreamTests.cs b/Test/UnrealStreamTests.cs index b11493d..6c6fe8a 100644 --- a/Test/UnrealStreamTests.cs +++ b/Test/UnrealStreamTests.cs @@ -1,9 +1,9 @@ using System; using System.IO; -using System.Reflection; using System.Text; using Microsoft.VisualStudio.TestTools.UnitTesting; using UELib; +using UELib.Branch; using UELib.Core; namespace Eliot.UELib.Test @@ -12,19 +12,19 @@ namespace Eliot.UELib.Test public class UnrealStreamTests { // HACK: Ugly workaround the issues with UPackageStream - private static UPackageStream CreateTempStream(string name = "test.u") + private static UPackageStream CreateTempStream() { - string tempFilePath = Path.Join(Assembly.GetExecutingAssembly().Location, "../", name); + string tempFilePath = Path.Join(Path.GetTempFileName()); File.WriteAllBytes(tempFilePath, BitConverter.GetBytes(UnrealPackage.Signature)); var stream = new UPackageStream(tempFilePath, FileMode.Open, FileAccess.ReadWrite); return stream; } - + [TestMethod] public void ReadString() { - using var stream = CreateTempStream("string.u"); + using var stream = CreateTempStream(); using var linker = new UnrealPackage(stream); linker.Summary = new UnrealPackage.PackageFileSummary { @@ -62,7 +62,7 @@ public void ReadString() [TestMethod] public void ReadAtomicStruct() { - using var stream = CreateTempStream("atomicstruct.u"); + using var stream = CreateTempStream(); using var linker = new UnrealPackage(stream); linker.Summary = new UnrealPackage.PackageFileSummary { @@ -73,7 +73,7 @@ public void ReadAtomicStruct() using var writer = new BinaryWriter(stream); // Skip past the signature writer.Seek(sizeof(int), SeekOrigin.Begin); - + // B, G, R, A; var inColor = new UColor(255, 128, 64, 80); stream.WriteAtomicStruct(ref inColor); @@ -82,11 +82,57 @@ public void ReadAtomicStruct() stream.Seek(sizeof(int), SeekOrigin.Begin); stream.ReadAtomicStruct(out UColor outColor); Assert.AreEqual(8, stream.Position); - + Assert.AreEqual(255, outColor.B); Assert.AreEqual(128, outColor.G); Assert.AreEqual(64, outColor.R); Assert.AreEqual(80, outColor.A); } + + [TestMethod] + public void TestBulkData() + { + using var stream = CreateTempStream(); + using var linker = new UnrealPackage(stream); + linker.Summary = new UnrealPackage.PackageFileSummary + { + Build = new UnrealPackage.GameBuild(linker), + }; + + using var writer = new BinaryWriter(stream); + // Skip past the signature + writer.Seek(sizeof(int), SeekOrigin.Begin); + + // Verify the oldest stage of LazyArray. + //linker.Summary.Version = (uint)PackageObjectLegacyVersion.LazyArraySkipCountToSkipOffset - 1; + //TestBulkDataSerialization(stream); + + //linker.Summary.Version = (uint)PackageObjectLegacyVersion.LazyArraySkipCountToSkipOffset; + //TestBulkDataSerialization(stream); + + linker.Summary.Version = (uint)PackageObjectLegacyVersion.LazyArrayReplacedWithBulkData; + TestBulkDataSerialization(stream); + } + + private void TestBulkDataSerialization(IUnrealStream stream) + { + byte[] rawData = Encoding.ASCII.GetBytes("LET'S PRETEND THAT THIS IS BULK DATA!"); + var bulkData = new UBulkData(0, rawData); + + long bulkPosition = stream.Position; + stream.Write(ref bulkData); + Assert.AreEqual(rawData.Length, bulkData.StorageSize); + + stream.Position = bulkPosition; + stream.Read(out UBulkData readBulkData); + Assert.IsNull(readBulkData.ElementData); + Assert.AreEqual(bulkData.StorageSize, readBulkData.StorageSize); + Assert.AreEqual(bulkData.StorageOffset, readBulkData.StorageOffset); + Assert.AreEqual(bulkData.ElementCount, readBulkData.ElementCount); + + readBulkData.LoadData(stream); + Assert.IsNotNull(readBulkData.ElementData); + Assert.AreEqual(bulkData.ElementData.Length, readBulkData.ElementData.Length); + } } } diff --git a/src/Branch/PackageObjectLegacyVersion.cs b/src/Branch/PackageObjectLegacyVersion.cs index b47a248..9ff97ae 100644 --- a/src/Branch/PackageObjectLegacyVersion.cs +++ b/src/Branch/PackageObjectLegacyVersion.cs @@ -10,7 +10,7 @@ public enum PackageObjectLegacyVersion ReturnExpressionAddedToReturnToken = 62, SphereExtendsPlane = 62, - LazyArraySkipCountToSkipOffset = 62, + LazyArraySkipCountChangedToSkipOffset = 62, CharRemapAddedToUFont = 69, @@ -18,9 +18,9 @@ public enum PackageObjectLegacyVersion /// FIXME: Unknown version. /// CastStringSizeTokenDeprecated = 70, - + PanUVRemovedFromPoly = 78, - + CompMipsDeprecated = 84, /// @@ -36,28 +36,34 @@ public enum PackageObjectLegacyVersion // The estimated version changes that came after the latest known UE2 build. TextureDeprecatedFromPoly = 170, MaterialAddedToPoly = 170, - + + UE3 = 178, + /// /// Present in all released UE3 games (starting with RoboBlitz). /// /// FIXME: Unknown version. /// IsLocalAddedToDelegateFunctionToken = 181, - - UE3 = 184, - + // FIXME: Version RangeConstTokenDeprecated = UE3, - + // FIXME: Version FastSerializeStructs = UE3, - + // FIXME: Version EnumTagNameAddedToBytePropertyTag = UE3, // 227 according to the GoW client FixedVerticesToArrayFromPoly = 227, + // Thanks to @https://www.gildor.org/ for reverse-engineering the lazy-loader version changes. + LazyLoaderFlagsAddedToLazyArray = 251, + StorageSizeAddedToLazyArray = 254, + L8AddedToLazyArray = 260, + LazyArrayReplacedWithBulkData = 266, + // FIXME: Not attested in the GoW client, must have been before v321 LightMapScaleRemovedFromPoly = 300, @@ -66,10 +72,10 @@ public enum PackageObjectLegacyVersion // 321 according to the GoW client ElementOwnerAddedToUPolys = 321, - + // 417 according to the GoW client LightingChannelsAddedToPoly = 417, - + VerticalOffsetAddedToUFont = 506, CleanupFonts = 511, @@ -88,4 +94,4 @@ public enum PackageObjectLegacyVersion ForceScriptOrderAddedToUClass = 749, SuperReferenceMovedToUStruct = 756, } -} \ No newline at end of file +} diff --git a/src/Core/Types/UBulkData.cs b/src/Core/Types/UBulkData.cs new file mode 100644 index 0000000..df18437 --- /dev/null +++ b/src/Core/Types/UBulkData.cs @@ -0,0 +1,277 @@ +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using UELib.Annotations; +using UELib.Branch; +using UELib.Flags; + +namespace UELib.Core +{ + /// + /// Implements FUntypedBulkData (UE3) and TLazyArray (UE3 or older) + /// + /// The primitive element type of the bulk data. + public struct UBulkData : IUnrealSerializableClass, IDisposable + where TElement : unmanaged + { + public BulkDataFlags Flags; + + public long StorageSize; + public long StorageOffset; + + public int ElementCount; + [CanBeNull] public byte[] ElementData; + + // TODO: multi-byte based data. + public UBulkData(BulkDataFlags flags, byte[] rawData) + { + Flags = flags; + StorageSize = -1; + StorageOffset = -1; + + ElementCount = rawData.Length; + ElementData = rawData; + } + + public void Deserialize(IUnrealStream stream) + { + if (stream.Version < (uint)PackageObjectLegacyVersion.LazyArrayReplacedWithBulkData) + { + DeserializeLegacyLazyArray(stream); + return; + } + + stream.Read(out uint flags); + Flags = (BulkDataFlags)flags; + + stream.Read(out ElementCount); + + StorageSize = stream.ReadInt32(); + StorageOffset = stream.ReadInt32(); + + if (Flags.HasFlag(BulkDataFlags.StoreInSeparateFile)) + { + return; + } + + Debug.Assert(stream.AbsolutePosition == StorageOffset); + // Skip the ElementData + stream.AbsolutePosition = StorageOffset + StorageSize; + } + + // Deserializes the TLazyArray format + private void DeserializeLegacyLazyArray(IUnrealStream stream) + { + int elementSize = Unsafe.SizeOf(); + + if (stream.Version < (uint)PackageObjectLegacyVersion.LazyArraySkipCountChangedToSkipOffset) + { + ElementCount = stream.ReadIndex(); + + StorageSize = ElementCount * elementSize; + StorageOffset = stream.AbsolutePosition; + + stream.AbsolutePosition += StorageSize; + return; + } + + // Absolute position in stream + long skipOffset = stream.ReadInt32(); + + // We still need these checks for Rainbow Six: Vegas 2 (V241) + if (stream.Version >= (uint)PackageObjectLegacyVersion.StorageSizeAddedToLazyArray) + { + StorageSize = stream.ReadInt32(); + } + + if (stream.Version >= (uint)PackageObjectLegacyVersion.LazyLoaderFlagsAddedToLazyArray) + { + stream.Read(out uint flags); + Flags = (BulkDataFlags)flags; + } + + if (stream.Version >= (uint)PackageObjectLegacyVersion.L8AddedToLazyArray) + { + stream.Read(out UName l8); + } + + ElementCount = stream.ReadLength(); + + StorageSize = skipOffset - stream.AbsolutePosition; + StorageOffset = stream.AbsolutePosition; + + stream.AbsolutePosition = skipOffset; + } + + public void LoadData(IUnrealStream stream) + { + if (Flags.HasFlag(BulkDataFlags.Unused) || ElementCount == 0) + { + return; + } + + int elementSize = Unsafe.SizeOf(); + ElementData = new byte[ElementCount * elementSize]; + + if (Flags.HasFlag(BulkDataFlags.StoreInSeparateFile)) + { + throw new NotSupportedException("Cannot read bulk data from a separate file yet!"); + } + + if (Flags.HasFlag(BulkDataFlags.StoreOnlyPayload)) + { + Debugger.Break(); + } + + long returnPosition = stream.Position; + stream.AbsolutePosition = StorageOffset; + + if (Flags.HasFlag(BulkDataFlags.Compressed)) + { + stream.ReadStruct(out CompressedChunkHeader header); + + Debugger.Break(); + + int offset = 0; + var decompressFlags = Flags.ToCompressionFlags(); + foreach (var chunk in header.Chunks) + { + stream.Read(ElementData, offset, chunk.CompressedSize); + offset += chunk.Decompress(ElementData, offset, decompressFlags); + } + + Debug.Assert(offset == ElementData.Length); + } + else + { + stream.Read(ElementData, 0, ElementData.Length); + } + + stream.Position = returnPosition; + } + + public void Serialize(IUnrealStream stream) + { + Debug.Assert(ElementData != null); + + if (stream.Version < (uint)PackageObjectLegacyVersion.LazyArrayReplacedWithBulkData) + { + SerializeLegacyLazyArray(stream); + return; + } + + stream.Write((uint)Flags); + + int elementCount = ElementData.Length / Unsafe.SizeOf(); + Debug.Assert(ElementCount == elementCount, "ElementCount mismatch"); + stream.Write(ElementCount); + + long storageSizePosition = stream.Position; + stream.Write((int)StorageSize); + stream.Write((int)StorageOffset); + + if (Flags.HasFlag(BulkDataFlags.StoreInSeparateFile)) + { + return; + } + + long storageOffsetPosition = stream.AbsolutePosition; + stream.Write(ElementData); + + StorageOffset = storageOffsetPosition; + StorageSize = stream.AbsolutePosition - StorageOffset; + + // Go back and rewrite the actual values. + stream.Position = storageSizePosition; + stream.Write((int)StorageSize); + stream.Write((int)StorageOffset); + // Restore + stream.Position = StorageOffset + StorageSize; + } + + private void SerializeLegacyLazyArray(IUnrealStream stream) + { + int elementSize = Unsafe.SizeOf(); + Debug.Assert(ElementCount == ElementData.Length / elementSize, "ElementCount mismatch"); + + if (stream.Version < (uint)PackageObjectLegacyVersion.LazyArraySkipCountChangedToSkipOffset) + { + stream.WriteIndex(ElementCount); + + StorageOffset = stream.AbsolutePosition; + stream.Write(ElementData, 0, ElementData.Length); + StorageSize = stream.AbsolutePosition - StorageOffset; + return; + } + + long storageDataPosition = stream.Position; + const int fakeSkipOffset = 0; + stream.Write(fakeSkipOffset); + + if (stream.Version >= (uint)PackageObjectLegacyVersion.StorageSizeAddedToLazyArray) + { + stream.Write((int)StorageSize); + } + + if (stream.Version >= (uint)PackageObjectLegacyVersion.LazyLoaderFlagsAddedToLazyArray) + { + stream.Write((uint)Flags); + } + + if (stream.Version >= (uint)PackageObjectLegacyVersion.L8AddedToLazyArray) + { + var dummy = new UName(""); + stream.Write(dummy); + } + + stream.WriteIndex(ElementCount); + StorageOffset = stream.AbsolutePosition; + if (Flags.HasFlag(BulkDataFlags.StoreInSeparateFile)) + { + throw new NotSupportedException("Cannot write bulk data to a separate file yet!"); + } + + if (Flags.HasFlag(BulkDataFlags.Compressed)) + { + throw new NotSupportedException("Compressing bulk data is not yet supported."); + } + + stream.Write(ElementData, 0, ElementData.Length); + + StorageSize = stream.AbsolutePosition - StorageOffset; + + long realSkipOffset = stream.AbsolutePosition; + stream.Position = storageDataPosition; + stream.Write((int)realSkipOffset); + stream.AbsolutePosition = realSkipOffset; + } + + public void Dispose() => ElementData = null; + } + + [Flags] + public enum BulkDataFlags : uint + { + // Formerly PayloadInSeparateFile? Attested in the assembly "!(LazyLoaderFlags & LLF_PayloadInSeparateFile)" + StoreInSeparateFile = 0b1, + CompressedZLIB = 0b10, + ForceSingleElementSerialization = 0b100, + SingleUse = 0b1000, + CompressedLZO = 0b10000, + Unused = 0b100000, + StoreOnlyPayload = 0b1000000, + CompressedLZX = 0b10000000, + Compressed = CompressedZLIB | CompressedLZO | CompressedLZX + } + + public static class BulkDataFlagsExtensions + { + public static CompressionFlags ToCompressionFlags(this BulkDataFlags flags) + { + uint cFlags = (((uint)flags & (uint)BulkDataFlags.CompressedZLIB) >> 1) + | (((uint)flags & (uint)BulkDataFlags.CompressedLZO) >> 3) + | (((uint)flags & (uint)BulkDataFlags.CompressedLZX) >> 5); + return (CompressionFlags)cFlags; + } + } +} diff --git a/src/UnrealPackageCompression.cs b/src/UnrealPackageCompression.cs index ee93a86..3824eec 100644 --- a/src/UnrealPackageCompression.cs +++ b/src/UnrealPackageCompression.cs @@ -1,5 +1,6 @@ using System.Diagnostics.CodeAnalysis; using UELib.Annotations; +using UELib.Flags; namespace UELib { @@ -127,5 +128,10 @@ public void Deserialize(IUnrealStream stream) CompressedSize = stream.ReadInt32(); UncompressedSize = stream.ReadInt32(); } + + public int Decompress(byte[] compressedData, int index, CompressionFlags flags) + { + return UncompressedSize; + } } -} \ No newline at end of file +} diff --git a/src/UnrealStream.cs b/src/UnrealStream.cs index 62b1393..9e93acc 100644 --- a/src/UnrealStream.cs +++ b/src/UnrealStream.cs @@ -983,28 +983,6 @@ public static void ReadMarshalArray(this IUnrealStream stream, out UArray #endif } - public static void ReadLazyArray(this IUnrealStream stream, out byte[] array) - { -#if BINARYMETADATA - long position = stream.Position; -#endif - if (stream.Version < (uint)PackageObjectLegacyVersion.LazyArraySkipCountToSkipOffset) - { - int skipCount = stream.ReadIndex(); - } - else - { - int skipOffset = stream.ReadInt32(); - } - - int c = stream.ReadLength(); - array = new byte[c]; - stream.Read(array, 0, c); -#if BINARYMETADATA - stream.LastPosition = position; -#endif - } - public static void ReadArray(this IUnrealStream stream, out UArray array) { #if BINARYMETADATA @@ -1188,6 +1166,13 @@ public static void Read(this IUnrealStream stream, out T value) value = ReadObject(stream); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Read(this IUnrealStream stream, out UBulkData value) + where T : unmanaged + { + ReadStruct(stream, out value); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Read(this IUnrealStream stream, out UObject value) { @@ -1305,32 +1290,6 @@ public static void WriteArray(this IUnrealStream stream, ref UArray array) } } - public static void WriteLazyArray(this IUnrealStream stream, ref byte[] array) - { - Debug.Assert(array != null); - if (stream.Version < (uint)PackageObjectLegacyVersion.LazyArraySkipCountToSkipOffset) - { - stream.WriteIndex(array.Length); - WriteIndex(stream, array.Length); - Write(stream, array, 0, array.Length); - } - else // skip using an absolute offset - { - var p = stream.Position; - Write(stream, 0); - - WriteIndex(stream, array.Length); - Write(stream, array, 0, array.Length); - - // We could easily predict the skip offset for byte arrays, - // but at some point we'll have to implement this for non-1byte arrays. - var p2 = stream.Position; - stream.Position = p; - Write(stream, (int)p2); - stream.Position = p2; - } - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void WriteStruct(this IUnrealStream stream, ref T item) where T : struct, IUnrealSerializableClass @@ -1383,6 +1342,13 @@ public static void Write(this IUnrealStream stream, UName name) stream.UW.Write((uint)name.Number + 1); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Write(this IUnrealStream stream, ref UBulkData value) + where T : unmanaged + { + WriteStruct(stream, ref value); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Write(this IUnrealStream stream, UObject obj) {