Deduplicate & Edit Meta

Penumbra/Game/RefreshActors.cs

@@ -0,0 +1,54 @@
+using System.Runtime.InteropServices;
+using Dalamud.Game.ClientState.Actors;
+using Dalamud.Game.ClientState.Actors.Types;
+using System.Threading.Tasks;
+
+namespace Penumbra
+{
+    public static class RefreshActors
+    {
+        private const int RenderModeOffset      = 0x0104;
+        private const int RenderTaskPlayerDelay = 75;
+        private const int RenderTaskOtherDelay  = 25;
+        private const int ModelInvisibilityFlag = 0b10;
+
+        private static async void Redraw(Actor actor)
+        {
+            var ptr = actor.Address;
+            var renderModePtr = ptr + RenderModeOffset;
+            var renderStatus = Marshal.ReadInt32(renderModePtr);
+
+            async void DrawObject(int delay)
+            {
+                Marshal.WriteInt32(renderModePtr, renderStatus | ModelInvisibilityFlag);
+                await Task.Delay(delay);
+                Marshal.WriteInt32(renderModePtr, renderStatus & ~ModelInvisibilityFlag);
+            }
+
+            if (actor.ObjectKind == Dalamud.Game.ClientState.Actors.ObjectKind.Player)
+            {
+                DrawObject(RenderTaskPlayerDelay);
+                await Task.Delay(RenderTaskPlayerDelay);
+            }
+            else
+                DrawObject(RenderTaskOtherDelay);
+
+        }
+
+        public static void RedrawSpecific(ActorTable actors, string name)
+        {
+            if (name?.Length == 0)
+                RedrawAll(actors);
+
+            foreach (var actor in actors)
+                if (actor.Name == name)
+                    Redraw(actor);
+        }
+
+        public static void RedrawAll(ActorTable actors)
+        {
+            foreach (var actor in actors)
+                Redraw(actor);
+        }
+    }
+}

Penumbra/Importer/TexToolsImport.cs

@@ -58,7 +58,10 @@ public void ImportModPack( FileInfo modPackFile )
 
                 case ".ttmp2":
                     ImportV2ModPack( modPackFile );
-                    return;
+                    return;
+
+                default:
+                    throw new ArgumentException( $"Unrecognized modpack format: {modPackFile.Extension}", nameof(modPackFile) );
             }
 
             State = ImporterState.Done;
@@ -237,19 +240,22 @@ private void AddMeta( DirectoryInfo baseFolder, DirectoryInfo groupFolder,ModGro
                 GroupName = group.GroupName,
                 Options = new List<Option>(),
             };
-            foreach( var opt in group.OptionList )
+            foreach( var opt in group.OptionList ) 
             {
                 var optio = new Option
                 {
                     OptionName = opt.Name,
-                    OptionDesc = String.IsNullOrEmpty( opt.Description ) ? "" : opt.Description,
-                    OptionFiles = new Dictionary<string, string>()
+                    OptionDesc = String.IsNullOrEmpty(opt.Description) ? "" : opt.Description,
+                    OptionFiles = new Dictionary<string, HashSet<string>>()
                 };
-                var optDir = new DirectoryInfo( Path.Combine( groupFolder.FullName, opt.Name ) );
-                foreach( var file in optDir.EnumerateFiles( "*.*", SearchOption.AllDirectories ) )
+                var optDir = new DirectoryInfo(Path.Combine( groupFolder.FullName, opt.Name));
+                if (optDir.Exists)
                 {
-                    optio.OptionFiles[file.FullName.Substring( baseFolder.FullName.Length ).TrimStart( '\\' )] = file.FullName.Substring( optDir.FullName.Length ).TrimStart( '\\' ).Replace( '\\', '/' );
-                }
+                    foreach ( var file in optDir.EnumerateFiles("*.*", SearchOption.AllDirectories) ) 
+                    {
+                        optio.AddFile(file.FullName.Substring(baseFolder.FullName.Length).TrimStart('\\'), file.FullName.Substring(optDir.FullName.Length).TrimStart('\\').Replace('\\','/'));
+                    }
+                }
                 Inf.Options.Add( optio );
             }
             meta.Groups.Add( group.GroupName, Inf );
@@ -333,4 +339,4 @@ private static string GetStringFromZipEntry( ZipFile file, ZipEntry entry, Encod
             return encoding.GetString( ms.ToArray() );
         }
     }
-}
+}

Penumbra/Models/Deduplicator.cs

@@ -0,0 +1,163 @@
+using System.Collections.Generic;
+using System.Security.Cryptography;
+using System.IO;
+using System.Linq;
+using System.Collections;
+using Dalamud.Plugin;
+
+namespace Penumbra.Models
+{
+    public class Deduplicator
+    {
+        private DirectoryInfo baseDir;
+        private int           baseDirLength;
+        private ModMeta       mod;
+        private SHA256        hasher = null;
+
+        private Dictionary<long, List<FileInfo>> filesBySize;
+
+        private ref SHA256 Sha()
+        {
+            if (hasher == null)
+                hasher = SHA256.Create();
+            return ref hasher;
+        }
+
+        public Deduplicator(DirectoryInfo baseDir, ModMeta mod)
+        {
+            this.baseDir       = baseDir;
+            this.baseDirLength = baseDir.FullName.Length;
+            this.mod           = mod;
+            filesBySize        = new();
+
+            BuildDict();  
+        }
+
+        private void BuildDict()
+        {
+            foreach( var file in baseDir.EnumerateFiles( "*.*", SearchOption.AllDirectories ) )
+            {
+                var fileLength = file.Length;
+                if (filesBySize.TryGetValue(fileLength, out var files))
+                    files.Add(file);
+                else
+                    filesBySize[fileLength] = new(){ file };
+            }
+        }
+
+        public void Run()
+        {
+            foreach (var pair in filesBySize)
+            {
+                if (pair.Value.Count < 2)
+                    continue;
+
+                if (pair.Value.Count == 2)
+                {
+                    if (CompareFilesDirectly(pair.Value[0], pair.Value[1]))
+                        ReplaceFile(pair.Value[0], pair.Value[1]);
+                }
+                else
+                {
+                    var deleted = Enumerable.Repeat(false, pair.Value.Count).ToArray();
+                    var hashes  = pair.Value.Select( F => ComputeHash(F)).ToArray();
+
+                    for (var i = 0; i < pair.Value.Count; ++i)
+                    {
+                        if (deleted[i])
+                            continue;
+
+                        for (var j = i + 1; j < pair.Value.Count; ++j)
+                        {
+                            if (deleted[j])
+                                continue;
+
+                            if (!CompareHashes(hashes[i], hashes[j]))
+                                continue;
+
+                            ReplaceFile(pair.Value[i], pair.Value[j]);
+                            deleted[j] = true;
+                        }
+                    }
+                }
+            }
+            ClearEmptySubDirectories(baseDir);
+        }
+
+        private void ReplaceFile(FileInfo f1, FileInfo f2)
+        {
+            var relName1 = f1.FullName.Substring(baseDirLength).TrimStart('\\');
+            var relName2 = f2.FullName.Substring(baseDirLength).TrimStart('\\');
+
+            var inOption = false;
+            foreach (var group in mod.Groups.Select( g => g.Value.Options))
+            {
+                foreach (var option in group)
+                {
+                    if (option.OptionFiles.TryGetValue(relName2, out var values))
+                    {
+                        inOption = true;
+                        foreach (var value in values)
+                            option.AddFile(relName1, value);
+                        option.OptionFiles.Remove(relName2);
+                    }
+                }
+            }
+            if (!inOption)
+            {
+                const string duplicates = "Duplicates";
+                if (!mod.Groups.ContainsKey(duplicates))
+                {
+                    InstallerInfo info = new()
+                    {
+                        GroupName = duplicates,
+                        SelectionType = SelectType.Single,
+                        Options = new()
+                        { 
+                            new()
+                            {
+                                OptionName  = "Required", 
+                                OptionDesc  = "", 
+                                OptionFiles = new()
+                            }
+                        }
+                    };
+                    mod.Groups.Add(duplicates, info);
+                }
+                mod.Groups[duplicates].Options[0].AddFile(relName1, relName2.Replace('\\', '/'));
+                mod.Groups[duplicates].Options[0].AddFile(relName1, relName1.Replace('\\', '/'));
+            }
+            PluginLog.Information($"File {relName1} and {relName2} are identical. Deleting the second.");
+            f2.Delete();
+        }
+
+        public static bool CompareFilesDirectly(FileInfo f1, FileInfo f2)
+        {
+            return File.ReadAllBytes(f1.FullName).SequenceEqual(File.ReadAllBytes(f2.FullName));
+        }
+
+        public static bool CompareHashes(byte[] f1, byte[] f2)
+        {
+            return StructuralComparisons.StructuralEqualityComparer.Equals(f1, f2);
+        }
+
+        public byte[] ComputeHash(FileInfo f)
+        {
+            var stream = File.OpenRead( f.FullName );
+            var ret = Sha().ComputeHash(stream);
+            stream.Dispose();
+            return ret;
+        }
+
+        // Does not delete the base directory itself even if it is completely empty at the end.
+        public static void ClearEmptySubDirectories(DirectoryInfo baseDir)
+        {
+            foreach (var subDir in baseDir.GetDirectories())
+            {
+                ClearEmptySubDirectories(subDir);
+                if (subDir.GetFiles().Length == 0 && subDir.GetDirectories().Length == 0)
+                    subDir.Delete();
+            }
+        }
+    }
+}

Penumbra/Models/GroupInformation.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using Newtonsoft.Json;
 
 namespace Penumbra.Models
 {
@@ -10,12 +11,24 @@ public struct Option
     {
         public string OptionName;
         public string OptionDesc;
-        public Dictionary<string, string> OptionFiles;
+
+        [JsonProperty(ItemConverterType = typeof(SingleOrArrayConverter<string>))]
+        public Dictionary<string, HashSet<string>> OptionFiles;
+
+        public bool AddFile(string filePath, string gamePath)
+        {
+            if (OptionFiles.TryGetValue(filePath, out var set))
+                return set.Add(gamePath);
+            else
+                OptionFiles[filePath] = new(){ gamePath };
+            return true;
+        }
     }
-    public struct InstallerInfo
-    {
+
+    public struct InstallerInfo {
         public string GroupName;
+        [JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))]
         public SelectType SelectionType;
         public List<Option> Options;
     }
-}
+}

Penumbra/Models/ModMeta.cs

@@ -1,4 +1,8 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+using System.Linq;
+using System.IO;
+using System;
 
 namespace Penumbra.Models
 {
@@ -17,6 +21,25 @@ public class ModMeta
 
         public Dictionary< string, string > FileSwaps { get; } = new();
 
-        public Dictionary<string, InstallerInfo> Groups { get; set; } = new();
+        public Dictionary<string, InstallerInfo> Groups { get; set; } = new();
+
+        [JsonIgnore]
+        public bool HasGroupWithConfig { get; set; } = false;
+
+        public static ModMeta LoadFromFile(string filePath)
+        {
+            try
+            {
+                var meta = JsonConvert.DeserializeObject< ModMeta >( File.ReadAllText( filePath ) );
+                meta.HasGroupWithConfig = meta.Groups != null && meta.Groups.Count > 0 
+                    && meta.Groups.Values.Any( G => G.SelectionType == SelectType.Multi || G.Options.Count > 1);
+                return meta;
+            }
+            catch( Exception)
+            {
+                return null;
+                // todo: handle broken mods properly
+            }
+        }
     }
 }

Penumbra/Mods/ModCollection.cs

@@ -64,19 +64,7 @@ public void Load( bool invertOrder = false )
                     continue;
                 }
 
-                ModMeta meta;
-
-                try
-                {
-                    meta = JsonConvert.DeserializeObject< ModMeta >( File.ReadAllText( metaFile.FullName ) );
-                }
-                catch( Exception e )
-                {
-                    PluginLog.Error( e, "failed to parse mod info, attempting basic load for: {FilePath}", metaFile.FullName );
-                    continue;
-
-                    // todo: handle broken mods properly
-                }
+                var meta = ModMeta.LoadFromFile(metaFile.FullName);
 
                 var mod = new ResourceMod
                 {
@@ -218,4 +206,4 @@ public IEnumerable<ResourceMod> GetOrderedAndEnabledModList( bool invertOrder =
                 .Select( x => (x.Mod, x) );
         }
     }
-}
+}