Skip to content

Commit

Permalink
[Performance] Significant Perf Improvement by MMIO & Fast PsbString C…
Browse files Browse the repository at this point in the history 
…ompare
  • Loading branch information
@UlyssesWu
UlyssesWu committed Jul 31, 2018
1 parent 0a07a11 commit 8a95395
Show file tree
Hide file tree
Showing 6 changed files with 226 additions and 71 deletions.
178 changes: 114 additions & 64 deletions FreeMote.Psb/Psb.cs
View file
Expand Up @@ -167,15 +167,24 @@ private void LoadFromStream(Stream stream)
{ {
stream.Seek(-4, SeekOrigin.Current); stream.Seek(-4, SeekOrigin.Current);
} }
BinaryReader br = new BinaryReader(stream, Encoding.UTF8);


BinaryReader sourceBr = new BinaryReader(stream, Encoding.UTF8);
BinaryReader br = sourceBr;

//Load Header //Load Header
Header = PsbHeader.Load(br); Header = PsbHeader.Load(br);
if (Header.IsHeaderEncrypted) if (Header.IsHeaderEncrypted)
{ {
throw new PsbBadFormatException(PsbBadFormatReason.Header); throw new PsbBadFormatException(PsbBadFormatReason.Header);
} }


//Switch MemoryMapped IO
if (PsbConstants.MemoryMappedLoading)
{
sourceBr.BaseStream.Position = 0;
br = new BinaryReader(new MemoryStream(sourceBr.ReadBytes((int)Header.OffsetChunkData)));
}

//Pre Load Strings //Pre Load Strings
br.BaseStream.Seek(Header.OffsetStrings, SeekOrigin.Begin); br.BaseStream.Seek(Header.OffsetStrings, SeekOrigin.Begin);
StringOffsets = new PsbArray(br.ReadByte() - (byte)PsbObjType.ArrayN1 + 1, br); StringOffsets = new PsbArray(br.ReadByte() - (byte)PsbObjType.ArrayN1 + 1, br);
Expand Down Expand Up @@ -229,6 +238,17 @@ private void LoadFromStream(Stream stream)
// var resArray = Unpack(br); // var resArray = Unpack(br);
//} //}


if (PsbConstants.MemoryMappedLoading)
{
br.Close();
br.Dispose();
}

//Load Resource
foreach (var res in Resources)
{
LoadResource(res, sourceBr);
}
Resources.Sort((r1, r2) => (int)((r1.Index ?? int.MaxValue) - (r2.Index ?? int.MaxValue))); Resources.Sort((r1, r2) => (int)((r1.Index ?? int.MaxValue) - (r2.Index ?? int.MaxValue)));
Type = InferType(); Type = InferType();
} }
Expand Down Expand Up @@ -327,14 +347,14 @@ private IPsbValue Unpack(BinaryReader br, bool lazyLoad = false)
var str = new PsbString(typeByte - (byte)PsbObjType.StringN1 + 1, br); var str = new PsbString(typeByte - (byte)PsbObjType.StringN1 + 1, br);
if (lazyLoad) if (lazyLoad)
{ {
var found = Strings.Find(s => s.Index != null && s.Index == str.Index); var foundStr = Strings.Find(s => s.Index != null && s.Index == str.Index);
if (found == null) if (foundStr == null)
{ {
Strings.Add(str); Strings.Add(str);
} }
else else
{ {
str = found; str = foundStr;
} }
} }
else else
Expand All @@ -347,21 +367,15 @@ private IPsbValue Unpack(BinaryReader br, bool lazyLoad = false)
case PsbObjType.ResourceN3: case PsbObjType.ResourceN3:
case PsbObjType.ResourceN4: case PsbObjType.ResourceN4:
var res = new PsbResource(typeByte - (byte)PsbObjType.ResourceN1 + 1, br); var res = new PsbResource(typeByte - (byte)PsbObjType.ResourceN1 + 1, br);
if (lazyLoad) //LoadResource(ref res, br); //No longer load Resources here
var foundRes = Resources.Find(r => r.Index == res.Index);
if (foundRes == null)
{ {
var found = Resources.Find(r => r.Index != null && r.Index == res.Index); Resources.Add(res);
if (found == null)
{
Resources.Add(res);
}
else
{
res = found;
}
} }
else else
{ {
LoadResource(ref res, br); res = foundRes;
} }
return res; return res;
case PsbObjType.Collection: case PsbObjType.Collection:
Expand All @@ -385,10 +399,10 @@ private IPsbValue Unpack(BinaryReader br, bool lazyLoad = false)
} }


/// <summary> /// <summary>
/// Load a dictionary /// Load a dictionary, won't ensure stream Position unless use <paramref name="lazyLoad"/>
/// </summary> /// </summary>
/// <param name="br"></param> /// <param name="br"></param>
/// <param name="lazyLoad"></param> /// <param name="lazyLoad">whether to lift stream Position to dictionary end</param>
/// <returns></returns> /// <returns></returns>
private PsbDictionary LoadObjects(BinaryReader br, bool lazyLoad = false) private PsbDictionary LoadObjects(BinaryReader br, bool lazyLoad = false)
{ {
Expand Down Expand Up @@ -437,10 +451,10 @@ private PsbDictionary LoadObjects(BinaryReader br, bool lazyLoad = false)
} }


/// <summary> /// <summary>
/// Load a collection /// Load a collection, won't ensure stream Position unless use <paramref name="lazyLoad"/>
/// </summary> /// </summary>
/// <param name="br"></param> /// <param name="br"></param>
/// <param name="lazyLoad"></param> /// <param name="lazyLoad">whether to lift stream Position</param>
/// <returns></returns> /// <returns></returns>
private PsbCollection LoadCollection(BinaryReader br, bool lazyLoad = false) private PsbCollection LoadCollection(BinaryReader br, bool lazyLoad = false)
{ {
Expand Down Expand Up @@ -474,7 +488,6 @@ private PsbCollection LoadCollection(BinaryReader br, bool lazyLoad = false)
{ {
endPos = br.BaseStream.Position; endPos = br.BaseStream.Position;
} }
//br.BaseStream.Seek(pos, SeekOrigin.Begin);
} }
if (lazyLoad) if (lazyLoad)
{ {
Expand All @@ -484,58 +497,72 @@ private PsbCollection LoadCollection(BinaryReader br, bool lazyLoad = false)
} }


/// <summary> /// <summary>
/// Load a resource based on index /// Load a resource content based on index, lift stream Position
/// </summary> /// </summary>
/// <param name="res"></param> /// <param name="res"></param>
/// <param name="br"></param> /// <param name="br"></param>
private void LoadResource(ref PsbResource res, BinaryReader br) private void LoadResource(PsbResource res, BinaryReader br)
{ {
if (res.Index == null) if (res.Index == null)
{ {
throw new IndexOutOfRangeException("Resource Index invalid"); throw new IndexOutOfRangeException("Resource Index invalid");
} }
//FIXED: Add check for re-used resources ////No longer used
var resIndex = res.Index; //var resIndex = res.Index;
var re = Resources.Find(r => r.Index == resIndex); //var re = Resources.Find(r => r.Index == resIndex);
if (re != null) //if (re != null)
{ //{
res = re; // res = re;
return; //Already loaded! // return; //Already loaded!
} //}
var pos = br.BaseStream.Position; //var pos = br.BaseStream.Position;
var offset = ChunkOffsets[(int)res.Index]; var offset = ChunkOffsets[(int)res.Index];
var length = ChunkLengths[(int)res.Index]; var length = ChunkLengths[(int)res.Index];
br.BaseStream.Seek(Header.OffsetChunkData + offset, SeekOrigin.Begin); br.BaseStream.Seek(Header.OffsetChunkData + offset, SeekOrigin.Begin);
res.Data = br.ReadBytes((int)length); res.Data = br.ReadBytes((int)length);
br.BaseStream.Seek(pos, SeekOrigin.Begin); //br.BaseStream.Seek(pos, SeekOrigin.Begin);
Resources.Add(res); //Resources.Add(res);
} }


/// <summary> /// <summary>
/// Load a string based on index /// Load a string based on index, lift stream Position
/// </summary> /// </summary>
/// <param name="str"></param> /// <param name="str"></param>
/// <param name="br"></param> /// <param name="br"></param>
private void LoadString(ref PsbString str, BinaryReader br) private void LoadString(ref PsbString str, BinaryReader br)
{ {
if (StringOffsets == null) //var pos = br.BaseStream.Position;
{
return;
}
var pos = br.BaseStream.Position;
Debug.Assert(str.Index != null, "Index can not be null"); Debug.Assert(str.Index != null, "Index can not be null");
br.BaseStream.Seek(Header.OffsetStringsData + StringOffsets[(int)str.Index], SeekOrigin.Begin); var idx = str.Index.Value;
var strValue = br.ReadStringZeroTrim(); PsbString refStr = null;
str.Value = strValue; if (Strings.Contains(str))
br.BaseStream.Seek(pos, SeekOrigin.Begin);
if (!Strings.Contains(str))
{ {
Strings.Add(str); refStr = Strings.Find(s => s.Index == idx);
if (PsbConstants.FastMode)
{
str = refStr;
return;
}

} }
else br.BaseStream.Seek(Header.OffsetStringsData + StringOffsets[(int)idx], SeekOrigin.Begin);
var strValue = br.ReadStringZeroTrim();

if (refStr != null && strValue == refStr.Value) //Strict value equal check
{ {
str = Strings.Find(s => s.Value == strValue); str = refStr;
return;
} }
str.Value = strValue;
//br.BaseStream.Seek(pos, SeekOrigin.Begin);
//if (!Strings.Contains(str))
//{
// Strings.Add(str);
//}
//else
//{
// str = Strings.Find(s => s.Value == strValue);
//}
} }


/// <summary> /// <summary>
Expand Down Expand Up @@ -876,8 +903,8 @@ public void SaveRawResources(string path)
} }


/// <summary> /// <summary>
/// Try skip header and load /// Try skip header and load. May (not) work on any PSB only if body is not encrypted
/// <para>May (not) work on any PSB only if body is not encrypted</para> /// <para><see cref="PsbConstants.MemoryMappedLoading"/> won't accelerate.</para>
/// <remarks>DuRaRaRa!!</remarks> /// <remarks>DuRaRaRa!!</remarks>
/// </summary> /// </summary>
/// <param name="path"></param> /// <param name="path"></param>
Expand Down Expand Up @@ -979,17 +1006,47 @@ private void LoadFromDullahan(Stream stream, int detectSize = 1024)
StringOffsets = new PsbArray(br.ReadByte() - (byte)PsbObjType.ArrayN1 + 1, br); StringOffsets = new PsbArray(br.ReadByte() - (byte)PsbObjType.ArrayN1 + 1, br);
Header.OffsetStringsData = (uint)br.BaseStream.Position; Header.OffsetStringsData = (uint)br.BaseStream.Position;
Strings.Sort((s1, s2) => (int)((s1.Index ?? int.MaxValue) - (s2.Index ?? int.MaxValue))); Strings.Sort((s1, s2) => (int)((s1.Index ?? int.MaxValue) - (s2.Index ?? int.MaxValue)));
for (var i = 0; i < Strings.Count; i++)
if (StringOffsets.Value.Count > 0 && PsbConstants.MemoryMappedLoading)
{
uint strsEndPos = StringOffsets.Value.Max();
br.BaseStream.Seek(strsEndPos, SeekOrigin.Current);
br.ReadStringZeroTrim();
strsEndPos = (uint)br.BaseStream.Position;
var strsLength = strsEndPos - Header.OffsetStringsData;
br.BaseStream.Seek(-strsLength, SeekOrigin.Current);

using (var strMs = new MemoryStream(br.ReadBytes((int)strsLength)))
using (var strBr = new BinaryReader(strMs))
{
for (var i = 0; i < Strings.Count; i++)
{
var str = Strings[i];
if (str.Index == null)
{
continue;
}
strBr.BaseStream.Seek(StringOffsets[(int)str.Index], SeekOrigin.Begin);
var strValue = strBr.ReadStringZeroTrim();
str.Value = strValue;
}
}
}
else
{ {
var str = Strings[i]; for (var i = 0; i < Strings.Count; i++)
if (str.Index == null)
{ {
continue; var str = Strings[i];
if (str.Index == null)
{
continue;
}
br.BaseStream.Seek(Header.OffsetStringsData + StringOffsets[(int)str.Index], SeekOrigin.Begin);
var strValue = br.ReadStringZeroTrim();
str.Value = strValue;
} }
br.BaseStream.Seek(Header.OffsetStringsData + StringOffsets[(int)str.Index], SeekOrigin.Begin);
var strValue = br.ReadStringZeroTrim();
str.Value = strValue;
} }



//Load Resources //Load Resources
while (br.PeekChar() != (int)PsbObjType.ArrayN1 && br.PeekChar() != (int)PsbObjType.ArrayN2) while (br.PeekChar() != (int)PsbObjType.ArrayN1 && br.PeekChar() != (int)PsbObjType.ArrayN2)
Expand Down Expand Up @@ -1041,14 +1098,7 @@ private void LoadFromDullahan(Stream stream, int detectSize = 1024)
Header.OffsetResourceOffsets = Header.OffsetChunkData; Header.OffsetResourceOffsets = Header.OffsetChunkData;
foreach (var res in Resources) foreach (var res in Resources)
{ {
if (res.Index == null) LoadResource(res, br);
{
continue;
}
var offset = ChunkOffsets[(int)res.Index];
var length = ChunkLengths[(int)res.Index];
br.BaseStream.Seek(Header.OffsetChunkData + offset, SeekOrigin.Begin);
res.Data = br.ReadBytes((int)length);
} }
} }


Expand Down
26 changes: 24 additions & 2 deletions FreeMote.Psb/PsbTypes.cs
View file
Expand Up @@ -689,12 +689,28 @@ public override string ToString()
} }
return s1.Equals(s2); return s1.Equals(s2);
} }

public static bool operator !=(PsbString s1, PsbString s2) public static bool operator !=(PsbString s1, PsbString s2)
{ {
return !(s1 == s2); return !(s1 == s2);
} }


public static bool operator ==(PsbString s1, string s2)
{
return s1?.Value == s2;
}
public static bool operator !=(PsbString s1, string s2)
{
return !(s1 == s2);
}
public static bool operator ==(string s1, PsbString s2)
{
return s1 == s2?.Value;
}
public static bool operator !=(string s1, PsbString s2)
{
return !(s1 == s2);
}

public override bool Equals(object obj) public override bool Equals(object obj)
{ {
var s = obj as PsbString; var s = obj as PsbString;
Expand All @@ -707,12 +723,18 @@ protected bool Equals(PsbString other)
{ {
return false; return false;
} }

//WARN: PsbString is considered Equal once they have same Index, rather than Value!
if (this.Index != null && other.Index != null)
{
return this.Index == other.Index;
}
return string.Equals(Value, other.Value); return string.Equals(Value, other.Value);
} }


public override int GetHashCode() public override int GetHashCode()
{ {
return (Value != null ? Value.GetHashCode() : 0); return Index != null ? (int) Index.Value : (Value != null ? Value.GetHashCode() : 0);
} }


public void WriteTo(BinaryWriter bw) public void WriteTo(BinaryWriter bw)
Expand Down
13 changes: 13 additions & 0 deletions FreeMote.Tests/PsBuildTest.cs
View file
Expand Up @@ -358,6 +358,19 @@ public void TestCompileMenuPsb()
PsbCompiler.CompileToFile(path, path + ".psbuild.psb", null, 2); PsbCompiler.CompileToFile(path, path + ".psbuild.psb", null, 2);
} }


[TestMethod]
public void TestMmioAndDullhanContent()
{
var resPath = Path.Combine(Environment.CurrentDirectory, @"..\..\Res");
var path = Path.Combine(resPath, "00_pro02.txt.scn");
var path2 = Path.Combine(resPath, "00_pro02.txt.scn.json");
//var path = Path.Combine(resPath, "akira_guide.psb");
//var path2 = Path.Combine(resPath, "akira_guide.psb.json");
//var psb = new PSB(path);
var psb = PSB.DullahanLoad(path);
var r = PsbDecompiler.Decompile(psb).Equals(File.ReadAllText(path2));
}

[TestMethod] [TestMethod]
public void TestCompareDecompile() public void TestCompareDecompile()
{ {
Expand Down

0 comments on commit 8a95395

Please sign in to comment.