Skip to content

Commit

Permalink
Added better FET / Fnv1 Support
Browse files Browse the repository at this point in the history
  • Loading branch information
@paulov-t
paulov-t committed May 1, 2023
1 parent afd0abb commit 55d3abe
Show file tree
Hide file tree
Showing 4 changed files with 159 additions and 26 deletions.
143 changes: 136 additions & 7 deletions Libraries/FMT.FileTools/Fnv1.cs
View file
Expand Up @@ -5,26 +5,155 @@ namespace FMT.FileTools
{
public static class Fnv1
{
public static int HashString(string s)
public static int HashString(string span)
{
//return (int)HashDepot.Fnv1.Hash32(Encoding.UTF8.GetBytes(s));
return (int)HashDepot.Fnv1a.Hash32(Encoding.UTF8.GetBytes(s));
return Fnv1a.HashString(span);
}
}

public static class Fnv1a
{
public static int HashString(string s)
public static int Hash(ReadOnlySpan<byte> data)
{
if (data.Length == 0)
{
return 5381;
}
uint hash = 5381u;
for (int i = 0; i < data.Length; i++)
{
hash = (hash * 33) ^ data[i];
}
return (int)hash;
}

public static int HashString(string span)
{
if (span == null)
{
throw new ArgumentNullException("span");
}
int length = Encoding.UTF8.GetByteCount(span);
Span<byte> span2 = ((length > 1024) ? ((Span<byte>)new byte[length]) : stackalloc byte[length]);
Span<byte> bytes = span2;
Encoding.UTF8.GetBytes(span, bytes);
return Hash(bytes);
}

public static int HashStringByHashDepot(string s)
{
return (int)HashDepot.Fnv1a.Hash32(Encoding.UTF8.GetBytes(s));
}
}

public static class Murmur2
{
public static UInt64 HashString64(string s, ulong seed)
private const ulong m = 14313749767032793493uL;

private const int r = 47;

public static ulong Hash64(byte[] data, ulong seed)
{
if (data == null)
{
throw new ArgumentNullException("data");
}
if (data.Length == 0)
{
return 0uL;
}
return Hash(data, seed);
}

public static ulong HashString64(string data, ulong seed)
{
if (data == null)
{
throw new ArgumentNullException("data");
}
return Hash64(Encoding.UTF8.GetBytes(data), seed);
}

private static ulong Hash(ushort key, ulong seed)
{
return Finish((seed ^ 0x8D494F26B7A3D32AuL ^ ((ulong)key << 8) ^ key) * 14313749767032793493uL);
}

private static ulong Hash(ulong key, ulong seed)
{
ulong h = seed ^ 0x35253C9ADE8F4CA8uL;
h = Mix(key, h);
return Finish(h);
}

private static ulong Hash(byte key, ulong seed)
{
return Finish((seed ^ 0xC6A4A7935BD1E995uL ^ key) * 14313749767032793493uL);
}

private unsafe static ulong Hash(byte[] key, ulong seed)
{
int len = key.Length;
ulong h = seed ^ (ulong)(len * -4132994306676758123L);
fixed (byte* ptr = &key[0])
{
int blocks = len / 8;
int remainder = len & 7;
ulong* b = (ulong*)ptr;
while (blocks-- > 0)
{
ulong* num = b;
b = num + 1;
h = Mix(*num, h);
}
byte* remainderBlock = (byte*)b;
switch (remainder)
{
default:
goto end_IL_001c;
case 7:
h ^= (ulong)(*(remainderBlock++)) << 48;
goto case 6;
case 6:
h ^= (ulong)(*(remainderBlock++)) << 40;
goto case 5;
case 5:
h ^= (ulong)(*(remainderBlock++)) << 32;
goto case 4;
case 4:
h ^= (ulong)(*(remainderBlock++)) << 24;
goto case 3;
case 3:
h ^= (ulong)(*(remainderBlock++)) << 16;
goto case 2;
case 2:
h ^= (ulong)(*(remainderBlock++)) << 8;
break;
case 1:
break;
}
h ^= *remainderBlock;
h *= 14313749767032793493uL;
end_IL_001c:;
}
return Finish(h);
}

private static ulong Mix(ulong k, ulong h)
{
k *= 14313749767032793493uL;
k ^= k >> 47;
k *= 14313749767032793493uL;
h ^= k;
h *= 14313749767032793493uL;
return h;
}

private static ulong Finish(ulong h)
{
return BitConverter.ToUInt64(HashDepot.MurmurHash3.Hash128(Encoding.UTF8.GetBytes(s), (uint)seed));
h ^= h >> 47;
h *= 14313749767032793493uL;
h ^= h >> 47;
return h;
}
}
}
30 changes: 16 additions & 14 deletions Libraries/FrostySdk/Frostbite/Compilers/BaseAssetCompiler.cs
View file
Expand Up @@ -414,10 +414,10 @@ public virtual void ModifyTOCChunks(string directory = "native_patch")
)
{
sbIndex++;
string tocFile = sbKey;
string tocFileSbKey = sbKey;
if (catalogInfo.SuperBundles[sbKey])
{
tocFile = sbKey.Replace("win32", catalogInfo.Name);
tocFileSbKey = sbKey.Replace("win32", catalogInfo.Name);
}

// Only handle Legacy stuff right now
Expand All @@ -426,30 +426,32 @@ public virtual void ModifyTOCChunks(string directory = "native_patch")
// continue;
//}

var pathToTOCFileRAW = $"{directory}/{tocFile}.toc";
var pathToTOCFileRAW = $"{directory}/{tocFileSbKey}.toc";
//string location_toc_file = FileSystem.Instance.ResolvePath(pathToTOCFileRAW);

var pathToTOCFile = FileSystem.Instance.ResolvePath(pathToTOCFileRAW, ModExecutor.UseModData);

using TOCFile tocFile2 = new TOCFile(pathToTOCFileRAW, false, false, true, sbIndex, true);
using TOCFile tocFileObj = new TOCFile(pathToTOCFileRAW, false, false, true, sbIndex, true);

// read the changed toc file in ModData
if (!tocFile2.TocChunks.Any())
if (!tocFileObj.TocChunks.Any())
continue;

if (!ModExecuter.ModifiedChunks.Any(x =>
x.Value.Bundles.Contains(tocFile2.ChunkDataBundleId)
x.Value.Bundles.Contains(tocFileObj.ChunkDataBundleId)
|| x.Value.Bundles.Contains(tocFileObj.ChunkDataBundleId_PreFMT2323)
|| x.Value.Bundles.Contains(Fnv1a.HashString("TocChunks"))
|| x.Value.Bundles.Contains(Fnv1a.HashStringByHashDepot("TocChunks"))
)
)
continue;

var patch = true;
var catalog = tocFile2.TocChunks.Max(x => x.ExtraData.Catalog.Value);
if (!tocFile2.TocChunks.Any(x => x.ExtraData.IsPatch))
var catalog = tocFileObj.TocChunks.Max(x => x.ExtraData.Catalog.Value);
if (!tocFileObj.TocChunks.Any(x => x.ExtraData.IsPatch))
patch = false;

var cas = tocFile2.TocChunks.Where(x => x.ExtraData.Catalog == catalog).Max(x => x.ExtraData.Cas.Value);
var cas = tocFileObj.TocChunks.Where(x => x.ExtraData.Catalog == catalog).Max(x => x.ExtraData.Cas.Value);

var newCas = cas;
//var nextCasPath = GetNextCasInCatalog(catalogInfo, cas, patch, out int newCas);
Expand All @@ -471,9 +473,9 @@ public virtual void ModifyTOCChunks(string directory = "native_patch")
{
foreach (var modChunk in ModExecuter.ModifiedChunks)
{
if (tocFile2.TocChunkGuids.Contains(modChunk.Key))
if (tocFileObj.TocChunkGuids.Contains(modChunk.Key))
{
var chunkIndex = tocFile2.TocChunks.FindIndex(x => x.Id == modChunk.Key
var chunkIndex = tocFileObj.TocChunks.FindIndex(x => x.Id == modChunk.Key
&& modChunk.Value.ModifiedEntry != null
//&& (modChunk.Value.ModifiedEntry.AddToTOCChunks || modChunk.Value.ModifiedEntry.AddToChunkBundle)
);
Expand All @@ -487,9 +489,9 @@ public virtual void ModifyTOCChunks(string directory = "native_patch")
if (data == null)
continue;

var chunkGuid = tocFile2.TocChunkGuids[chunkIndex];
var chunkGuid = tocFileObj.TocChunkGuids[chunkIndex];

var chunk = tocFile2.TocChunks[chunkIndex];
var chunk = tocFileObj.TocChunks[chunkIndex];
//DbObject dboChunk = tocFile2.TocChunkInfo[modChunk.Key];

nw_cas.Position = nw_cas.Length;
Expand All @@ -504,7 +506,7 @@ public virtual void ModifyTOCChunks(string directory = "native_patch")
IsPatch = patch,
};

nw_toc.Position = tocFile2.TocChunkPatchPositions[chunkGuid];// dboChunk.GetValue<long>("patchPosition");
nw_toc.Position = tocFileObj.TocChunkPatchPositions[chunkGuid];// dboChunk.GetValue<long>("patchPosition");
nw_toc.Write(Convert.ToByte(patch ? 1 : 0));
nw_toc.Write(Convert.ToByte(catalog));
nw_toc.Write(Convert.ToByte(newCas));
Expand Down
11 changes: 6 additions & 5 deletions Libraries/FrostySdk/Frostbite/Compilers/ManifestBundleAction.cs
View file
@@ -1,8 +1,10 @@
//using FMT.FileTools;
//using FrostySdk.IO;
//using FrostySdk.Managers;
//using ModdingSupport;
//using System;
//using System.Collections.Generic;
//using System.IO;
//using System.Linq;
//using System.Text;
//using System.Threading;
Expand All @@ -11,8 +13,7 @@

//namespace FrostySdk.Frostbite.Compilers
//{

// private class ManifestBundleAction
// public class ManifestBundleAction
// {
// private static readonly object resourceLock = new object();

Expand Down Expand Up @@ -82,8 +83,8 @@
// for (int i = 1; i < manifestBundle.files.Count; i++)
// {
// ManifestFileInfo manifestFileInfo2 = manifestBundle.files[i];
// int key = Fnv1a.HashString(fs.ResolvePath((manifestFileInfo2.file.IsInPatch ? "native_patch/" : "native_data/") + fs.GetCatalog(manifestFileInfo2.file) + "/cas.cat").ToLower());
// Dictionary<uint, CatResourceEntry> dictionary = parent.resources[key][manifestFileInfo2.file.CasIndex];
// int hashedPath = Fnv1a.HashString(fs.ResolvePath((manifestFileInfo2.file.IsInPatch ? "native_patch/" : "native_data/") + fs.GetCatalog(manifestFileInfo2.file) + "/cas.cat").ToLower());
// Dictionary<uint, CatResourceEntry> dictionary = parent.Resources[hashedPath][manifestFileInfo2.file.CasIndex];
// List<uint> list2 = dictionary.Keys.ToList();
// uint num = 0u;
// uint num2 = manifestFileInfo2.offset;
Expand Down Expand Up @@ -179,7 +180,7 @@
// if (handlerExtraData != null)
// {
// byte[] outData = null;
// Stream resourceData = parent.rm.GetResourceData(parent.fs.GetFilePath(manifestFileInfo5.file.CatalogIndex, manifestFileInfo5.file.CasIndex, manifestFileInfo5.file.IsInPatch), manifestFileInfo5.offset, manifestFileInfo5.size);
// Stream resourceData = AssetManager.Instance.GetResourceData(parent.fs.GetFilePath(manifestFileInfo5.file.CatalogIndex, manifestFileInfo5.file.CasIndex, manifestFileInfo5.file.IsInPatch), manifestFileInfo5.offset, manifestFileInfo5.size);
// ResAssetEntry resAssetEntry2 = (ResAssetEntry)handlerExtraData.Handler.Modify(resAssetEntry, resourceData, handlerExtraData.Data, out outData);
// if (!parent.archiveData.ContainsKey(resAssetEntry2.Sha1))
// {
Expand Down
1 change: 1 addition & 0 deletions Libraries/FrostySdk/Frostbite/PluginInterfaces/TOCFile.cs
View file
Expand Up @@ -51,6 +51,7 @@ public enum AssetUsage : byte

public string ChunkDataBundleName { get { return $"{NativeFileLocation}-TOC"; } }
public int ChunkDataBundleId { get { return Fnv1a.HashString(ChunkDataBundleName); } }
public int ChunkDataBundleId_PreFMT2323 { get { return Fnv1a.HashStringByHashDepot(ChunkDataBundleName); } }


/// <summary>
Expand Down

0 comments on commit 55d3abe

Please sign in to comment.