Add support for console/legacy QAR extraction and console FPK extract…

…ion/creation

Changes:
- Support legacy QAR formats (version 1 = GZ PC, version 2 = TPP PS3/360, version 3 = TPP PC)
- Add X360Reader / X360Writer classes for big endian FPK support
- Edit FPK classes to use these reader/writers instead of passing streams around
- Detect endianness based on the platform of the FPK and read/write using that endianness automatically

Console QAR and FPK files can be extracted perfectly (maybe 1% of files extract wrong though, unsure why..)
Creating console FPK files also seems to work fine, haven't worked on console/legacy QAR creation yet though.

GzsTool/Fpk/FpkEntry.cs

@@ -17,10 +17,10 @@ public FpkEntry()
         }
 
         [XmlIgnore]
-        public uint DataOffset { get; set; }
+        public long DataOffset { get; set; }
 
         [XmlIgnore]
-        public int DataSize { get; set; }
+        public ulong DataSize { get; set; }
 
         [XmlIgnore]
         public FpkString FilePathFpkString { get; set; }
@@ -47,21 +47,18 @@ public bool ShouldSerializeMd5Hash()
             return FilePathFpkString.ValueResolved == false;
         }
 
-        public static FpkEntry ReadFpkEntry(Stream input)
+        public static FpkEntry ReadFpkEntry(X360Reader reader)
         {
             FpkEntry fpkEntry = new FpkEntry();
-            fpkEntry.Read(input);
+            fpkEntry.Read(reader);
             return fpkEntry;
         }
 
-        private void Read(Stream input)
+        private void Read(X360Reader reader)
         {
-            BinaryReader reader = new BinaryReader(input, Encoding.Default, true);
-            DataOffset = reader.ReadUInt32();
-            reader.Skip(4);
-            DataSize = reader.ReadInt32();
-            reader.Skip(4);
-            FpkString fileName = FpkString.ReadFpkString(input);
+            DataOffset = reader.ReadInt64();
+            DataSize = reader.ReadUInt64();
+            FpkString fileName = FpkString.ReadFpkString(reader);
             Md5Hash = reader.ReadBytes(16);
             fileName.ResolveString(Md5Hash);
             FilePathFpkString = fileName;
@@ -82,7 +79,7 @@ private Stream ReadData(Stream input)
         {
             input.Position = DataOffset;
             byte[] result = new byte[DataSize];
-            input.Read(result, 0, DataSize);
+            input.Read(result, 0, (int)DataSize);
             return new MemoryStream(result);
         }
 
@@ -100,30 +97,27 @@ private string GetFpkEntryFileName()
             return fileName;
         }
 
-        public void WriteFilePath(Stream output)
+        public void WriteFilePath(X360Writer writer)
         {
             if (Md5Hash == null)
                 Md5Hash = Hashing.Md5HashText(FilePath);
-            FilePathFpkString.WriteString(output);
+            FilePathFpkString.WriteString(writer);
         }
 
-        public void Write(Stream output)
+        public void Write(X360Writer writer)
         {
-            BinaryWriter writer = new BinaryWriter(output, Encoding.Default, true);
             writer.Write(DataOffset);
-            writer.WriteZeros(4);
             writer.Write(DataSize);
-            writer.WriteZeros(4);
-            FilePathFpkString.Write(output);
+            FilePathFpkString.Write(writer);
             writer.Write(Md5Hash);
         }
 
-        public void WriteData(Stream output, IDirectory inputDirectory)
+        public void WriteData(X360Writer writer, IDirectory inputDirectory)
         {
-            DataOffset = (uint) output.Position;
+            DataOffset = (uint) writer.BaseStream.Position;
             byte[] data = inputDirectory.ReadFile(GetFpkEntryFileName());
-            DataSize = data.Length;
-            output.Write(data, 0, data.Length);
+            DataSize = (ulong)data.Length;
+            writer.BaseStream.Write(data, 0, data.Length);
         }
 
         public FileDataStreamContainer Export(Stream input)

GzsTool/Fpk/FpkFile.cs

@@ -5,12 +5,15 @@
 using System.Xml.Serialization;
 using GzsTool.Common;
 using GzsTool.Common.Interfaces;
+using GzsTool.Utility;
 
 namespace GzsTool.Fpk
 {
     [XmlType("FpkFile")]
     public class FpkFile : ArchiveFile
     {
+        private const int FpkMagicNumber = 0x66786F66; // foxf
+
         public FpkFile()
         {
             Entries = new List<FpkEntry>();
@@ -23,6 +26,12 @@ public FpkFile()
         [XmlAttribute("FpkType")]
         public FpkType FpkType { get; set; }
 
+        [XmlAttribute("Platform")]
+        public string Platform { get; set; }
+
+        [XmlAttribute("UnknownValue")]
+        public uint UnknownValue { get; set; }
+
         [XmlArray("Entries")]
         public List<FpkEntry> Entries { get; private set; }
 
@@ -36,29 +45,39 @@ public static FpkFile ReadFpkFile(Stream input)
             return fpkFile;
         }
 
+        public static bool IsBigEndianPlatform(string platform)
+        {
+            return platform == "x36" || platform == "ps3";
+        }
+
         public override void Read(Stream input)
         {
-            BinaryReader reader = new BinaryReader(input, Encoding.Default, true);
+            X360Reader reader = new X360Reader(input, Encoding.Default, true, false);
             uint magicNumber1 = reader.ReadUInt32(); // foxf
+            if (magicNumber1 != FpkMagicNumber)
+                return;
+
             ushort magicNumber2 = reader.ReadUInt16(); // pk
             FpkType = (FpkType) reader.ReadByte(); // ' ' or 'd'
-            byte magicNumber3 = reader.ReadByte(); // w
-            ushort magicNumber4 = reader.ReadUInt16(); // in
+            Platform = reader.ReadAsciiString(3);
             uint fileSize = reader.ReadUInt32();
             reader.Skip(18);
-            uint magicNumber5 = reader.ReadUInt32(); // 2
+
+            reader.FlipEndian = IsBigEndianPlatform(Platform);
+
+            UnknownValue = reader.ReadUInt32(); // 2 (4 on some console fpks?)
             uint fileCount = reader.ReadUInt32();
             uint referenceCount = reader.ReadUInt32();
             reader.Skip(4);
 
             for (int i = 0; i < fileCount; i++)
             {
-                Entries.Add(FpkEntry.ReadFpkEntry(input));
+                Entries.Add(FpkEntry.ReadFpkEntry(reader));
             }
 
             for (int i = 0; i < referenceCount; i++)
             {
-                References.Add(FpkReference.ReadFpkReference(input));
+                References.Add(FpkReference.ReadFpkReference(reader));
             }
         }
 
@@ -69,7 +88,7 @@ public override IEnumerable<FileDataStreamContainer> ExportFiles(Stream input)
 
         public override void Write(Stream output, IDirectory inputDirectory)
         {
-            BinaryWriter writer = new BinaryWriter(output, Encoding.Default, true);
+            X360Writer writer = new X360Writer(output, Encoding.Default, true, IsBigEndianPlatform(Platform));
             const int headerSize = 48;
             int indicesSize = 48*Entries.Count;
             int referenceSize = 16*References.Count;
@@ -79,43 +98,47 @@ public override void Write(Stream output, IDirectory inputDirectory)
 
             foreach (var fpkEntry in Entries)
             {
-                fpkEntry.WriteFilePath(output);
+                fpkEntry.WriteFilePath(writer);
             }
             foreach (var fpkReference in References)
             {
-                fpkReference.WriteFilePath(output);
+                fpkReference.WriteFilePath(writer);
             }
             output.AlignWrite(16, 0x00);
 
             foreach (var fpkEntry in Entries)
             {
-                fpkEntry.WriteData(output, inputDirectory);
+                fpkEntry.WriteData(writer, inputDirectory);
                 output.AlignWrite(16, 0x00);
             }
 
             uint fileSize = (uint) output.Position;
 
             output.Position = startPosition;
 
+            if (writer.FlipEndian)
+                writer.FlipEndian = false;
             writer.Write(0x66786f66); // foxf
             writer.Write((ushort) 0x6B70); //pk
             writer.Write((byte) FpkType);
-            writer.Write((byte) 0x77); // w
-            writer.Write((ushort) 0x6E69); // in
+            writer.WriteAsciiString(Platform, 3);
             writer.Write(fileSize);
+
+            writer.FlipEndian = IsBigEndianPlatform(Platform);
+
             writer.WriteZeros(18);
-            writer.Write(0x00000002);
+            writer.Write(UnknownValue);
             writer.Write(Entries.Count);
             writer.Write(References.Count);
             writer.WriteZeros(4);
 
             foreach (var fpkEntry in Entries)
             {
-                fpkEntry.Write(output);
+                fpkEntry.Write(writer);
             }
             foreach (var fpkReference in References)
             {
-                fpkReference.Write(output);
+                fpkReference.Write(writer);
             }
         }
     }

GzsTool/Fpk/FpkReference.cs

@@ -1,5 +1,6 @@
 using System.IO;
 using System.Xml.Serialization;
+using GzsTool.Utility;
 
 namespace GzsTool.Fpk
 {
@@ -21,26 +22,26 @@ public string FilePath
             set { ReferenceFilePath.Value = value; }
         }
 
-        public static FpkReference ReadFpkReference(Stream input)
+        public static FpkReference ReadFpkReference(X360Reader reader)
         {
             FpkReference fpkReference = new FpkReference();
-            fpkReference.Read(input);
+            fpkReference.Read(reader);
             return fpkReference;
         }
 
-        private void Read(Stream input)
+        private void Read(X360Reader reader)
         {
-            ReferenceFilePath = FpkString.ReadFpkString(input);
+            ReferenceFilePath = FpkString.ReadFpkString(reader);
         }
 
-        public void WriteFilePath(Stream output)
+        public void WriteFilePath(X360Writer writer)
         {
-            ReferenceFilePath.WriteString(output);
+            ReferenceFilePath.WriteString(writer);
         }
 
-        public void Write(Stream output)
+        public void Write(X360Writer writer)
         {
-            ReferenceFilePath.Write(output);
+            ReferenceFilePath.Write(writer);
         }
     }
 }

GzsTool/Fpk/FpkString.cs

@@ -9,7 +9,7 @@ public class FpkString
     {
         public string Value { get; set; }
         public byte[] EncryptedValue { get; set; }
-        public int StringOffset { get; set; }
+        public long StringOffset { get; set; }
         public int StringLength { get; set; }
         public bool ValueResolved { get; set; }
 
@@ -18,25 +18,23 @@ public bool ValueEncrypted
             get { return EncryptedValue != null; }
         }
 
-        public static FpkString ReadFpkString(Stream input)
+        public static FpkString ReadFpkString(X360Reader reader)
         {
             FpkString fpkString = new FpkString();
-            fpkString.Read(input);
+            fpkString.Read(reader);
             return fpkString;
         }
 
-        private void Read(Stream input)
+        private void Read(X360Reader reader)
         {
-            BinaryReader reader = new BinaryReader(input, Encoding.Default, true);
-            StringOffset = reader.ReadInt32();
-            reader.Skip(4);
+            StringOffset = reader.ReadInt64();
             StringLength = reader.ReadInt32();
             reader.Skip(4);
 
-            long endPosition = input.Position;
-            input.Position = StringOffset;
+            long endPosition = reader.BaseStream.Position;
+            reader.BaseStream.Position = StringOffset;
             Value = reader.ReadString(StringLength);
-            input.Position = endPosition;
+            reader.BaseStream.Position = endPosition;
         }
 
         public override string ToString()
@@ -64,20 +62,17 @@ public void ResolveString(byte[] md5Hash)
             ValueResolved = resolved;
         }
 
-        public void WriteString(Stream output)
+        public void WriteString(X360Writer writer)
         {
-            BinaryWriter writer = new BinaryWriter(output, Encoding.Default, true);
-            StringOffset = (int) output.Position;
+            StringOffset = (int) writer.BaseStream.Position;
             string value = ValueEncrypted ? Encoding.Default.GetString(EncryptedValue) : Value;
             StringLength = value.Length;
             writer.WriteNullTerminatedString(value);
         }
 
-        public void Write(Stream output)
+        public void Write(X360Writer writer)
         {
-            BinaryWriter writer = new BinaryWriter(output, Encoding.Default, true);
             writer.Write(StringOffset);
-            writer.WriteZeros(4);
             writer.Write(StringLength);
             writer.WriteZeros(4);
         }

GzsTool/GzsTool.csproj

@@ -67,12 +67,12 @@
     <Compile Include="Pftxs\PftxsFtexsFileEntry.cs" />
     <Compile Include="Qar\QarEntry.cs" />
     <Compile Include="Qar\QarFile.cs" />
-    <Compile Include="Utility\BigEndianBinaryReader.cs" />
     <Compile Include="Fpk\FpkEntry.cs" />
     <Compile Include="Fpk\FpkFile.cs" />
     <Compile Include="Fpk\FpkReference.cs" />
     <Compile Include="Fpk\FpkString.cs" />
     <Compile Include="Utility\Compression.cs" />
+    <Compile Include="Utility\Encryption.cs" />
     <Compile Include="Utility\Hashing.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -82,6 +82,7 @@
     <Compile Include="PathId\PathIdFile.cs" />
     <Compile Include="PathId\StringIndex.cs" />
     <Compile Include="Utility\StructuralEqualityComparer.cs" />
+    <Compile Include="Utility\X360IO.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config" />

GzsTool/PathId/FileHash.cs

@@ -17,7 +17,7 @@ public static FileHash ReadFileHash(Stream input)
 
         private void Read(Stream input)
         {
-            BigEndianBinaryReader reader = new BigEndianBinaryReader(input, Encoding.Default, true);
+            X360Reader reader = new X360Reader(input, Encoding.Default, true, true);
             Hash = reader.ReadUInt64();
         }
     }

GzsTool/PathId/FolderInfo.cs

@@ -18,7 +18,7 @@ public static FolderInfo ReadFolderInfo(Stream input)
 
         private void Read(Stream input)
         {
-            BigEndianBinaryReader reader = new BigEndianBinaryReader(input, Encoding.Default, true);
+            X360Reader reader = new X360Reader(input, Encoding.Default, true, true);
             Offset1 = reader.ReadUInt16();
             Offset2 = reader.ReadUInt16();
         }

GzsTool/PathId/PathIdFile.cs

@@ -33,7 +33,7 @@ public static PathIdFile ReadPathIdFile(Stream input)
 
         public void Read(Stream input)
         {
-            BigEndianBinaryReader reader = new BigEndianBinaryReader(input, Encoding.Default, true);
+            X360Reader reader = new X360Reader(input, Encoding.Default, true, true);
             int unknown1 = reader.ReadInt32();
             int hashCount = reader.ReadInt32();
             int folderCount = reader.ReadInt32();

GzsTool/PathId/StringIndex.cs

@@ -17,7 +17,7 @@ public static StringIndex ReadStringIndex(Stream input)
 
         private void Read(Stream input)
         {
-            BigEndianBinaryReader reader = new BigEndianBinaryReader(input, Encoding.Default, true);
+            X360Reader reader = new X360Reader(input, Encoding.Default, true, true);
             Offset = reader.ReadUInt32();
         }
     }