Merge pull request #12 from ovska/tmp

add traceability to Resources

Directory.Build.props

@@ -1,7 +1,7 @@
 <Project>
 
   <PropertyGroup>
-    <Version>0.1.4</Version>
+    <Version>0.1.5</Version>
     <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>

Pack3r.Console/Program.cs

@@ -95,7 +95,7 @@ await using (destination)
             app.Logger.Drain();
 
             string missingMsg = missingCount > 0
-                ? $" with {missingCount} file(s) not found"
+                ? $", with {missingCount} file{(missingCount != 1 ? "s" : "")} missing"
                 : "";
 
             if (!options.DryRun)

Pack3r.Console/Properties/launchSettings.json

@@ -10,11 +10,11 @@
     },
     "sungilarity d": {
       "commandName": "Project",
-      "commandLineArgs": "\"C:\\Temp\\ET\\map\\ET\\etmain\\maps\\sungilarity.map\" -d -i"
+      "commandLineArgs": "\"C:\\Temp\\ET\\map\\ET\\etmain\\maps\\sungilarity.map\" -d -i -e pak0.pk3 void.pk3dir"
     },
     "sungilarity src": {
       "commandName": "Project",
-      "commandLineArgs": "\"C:\\Temp\\ET\\map\\ET\\etmain\\maps\\sungilarity.map\" -d -i -s"
+      "commandLineArgs": "\"C:\\Temp\\ET\\map\\ET\\etmain\\maps\\sungilarity.map\" -d -i -rd"
     }
   }
 }

Pack3r.Core/IO/AssetSource.cs

@@ -13,6 +13,10 @@ public abstract class AssetSource : IDisposable
     public abstract string RootPath { get; }
     public abstract FileInfo? GetShaderlist();
 
+    public string Name => _name ??= Path.GetFileName(RootPath);
+
+    private string? _name;
+
     internal protected Dictionary<ReadOnlyMemory<char>, IAsset> Assets => _assetsLazy.Value;
 
     private readonly Lazy<Dictionary<ReadOnlyMemory<char>, IAsset>> _assetsLazy;
@@ -36,6 +40,7 @@ public virtual void Dispose()
     public bool TryHandleAsset(
         ZipArchive destination,
         ReadOnlyMemory<char> relativePath,
+        Resource resource,
         out ZipArchiveEntry? entry)
     {
         ObjectDisposedException.ThrowIf(_disposed, this);
@@ -48,7 +53,9 @@ public virtual void Dispose()
             }
             else
             {
-                _checker.CheckIntegrity(asset);
+                if (!resource.SourceOnly)
+                    _checker.CheckIntegrity(asset);
+
                 entry = asset.CreateEntry(destination);
             }
 

Pack3r.Core/IO/Line.cs

@@ -1,4 +1,6 @@
-namespace Pack3r.IO;
+using Pack3r.Extensions;
+
+namespace Pack3r.IO;
 
 public readonly struct Line : IEquatable<Line>
 {

Pack3r.Core/Models/Map.cs

@@ -2,6 +2,7 @@
 using System.Collections.Immutable;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.Globalization;
 using Pack3r.Extensions;
 using Pack3r.IO;
 using Pack3r.Services;
@@ -52,28 +53,6 @@ public Map(PackOptions options, IIntegrityChecker integrityChecker)
     private readonly Lazy<ImmutableArray<DirectoryInfo>> _assetDirs;
     private readonly Lazy<ImmutableArray<AssetSource>> _assetSrcs;
 
-    private readonly ConcurrentBag<Resource> _allResources = [];
-
-    public void LogResource(in Resource resource)
-    {
-        if (_options.ReferenceDebug)
-            _allResources.Add(resource);
-    }
-
-    public IEnumerable<Resource> TryGetAllResources()
-    {
-        if (_options.ReferenceDebug)
-        {
-            return _allResources
-                .OrderBy(r => r.Source, StringComparer.OrdinalIgnoreCase)
-                .ThenBy(r => r.Line)
-                .ThenBy(r => r.IsShader)
-                .ThenBy(r => r.Value, ROMCharComparer.Instance);
-        }
-
-        return [];
-    }
-
     /// <summary>
     /// Gets the relative etmain of the map.<br/>
     /// <c>ET/etmain/maps/file.map</c> -> <c>ET/etmain/</c>
@@ -101,7 +80,7 @@ public string GetMapRoot()
         throw new UnreachableException($"Could not get map root: {Path}");
     }
 
-    public string GetRelativeToRoot(string path) => IOPath.GetRelativePath(GetMapRoot(), path);
+    public string GetRelativeToRoot(string path) => IOPath.GetRelativePath(GetMapRoot(), path).NormalizePath();
 
     /// <summary>
     /// Returns path <strong>relative to ETMain</strong>.
@@ -163,7 +142,7 @@ private ImmutableArray<AssetSource> InitAssetSources()
 
             list.Add(new DirectoryAssetSource(dir, isExcluded: dirFilter == SourceFilter.Excluded, _integrityChecker));
 
-            // TODO: exclude all child pk3s in a pk3dir?
+            // pk3s need to be loaded always if there are pk3/dir excludes
             if (_options.LoadPk3s || _options.ExcludeSources.Count > 0)
             {
                 foreach (var file in dir.EnumerateFiles("*.pk3", SearchOption.TopDirectoryOnly))
@@ -173,8 +152,13 @@ private ImmutableArray<AssetSource> InitAssetSources()
                     if (pk3Filter == SourceFilter.Ignored)
                         continue;
 
-                    if (_options.LoadPk3s || pk3Filter == SourceFilter.Excluded)
-                        list.Add(new Pk3AssetSource(file.FullName, pk3Filter == SourceFilter.Excluded, _integrityChecker));
+                    // exclude all pk3s in an excluded pk3dir
+                    bool pk3isExcluded = dir.FullName.HasExtension("pk3dir")
+                        ? (dirFilter == SourceFilter.Excluded || pk3Filter == SourceFilter.Excluded)
+                        : pk3Filter == SourceFilter.Excluded;
+
+                    if (_options.LoadPk3s || pk3isExcluded)
+                        list.Add(new Pk3AssetSource(file.FullName, isExcluded: pk3isExcluded, _integrityChecker));
                 }
             }
         }
@@ -184,15 +168,15 @@ private ImmutableArray<AssetSource> InitAssetSources()
             1. pak0 is always first (and other excluded sources)
             2. all other pk3s are always last, in reverse alphabetical order
         */
+#pragma warning disable IDE0305 // Simplify collection initialization
+        byte[] sortKeys = new byte[512];
+
         return list
-            .OrderBy(s => s switch
-            {
-                DirectoryAssetSource d => AssetDirectories.IndexOf(d.Directory),
-                Pk3AssetSource p => p.IsExcluded ? int.MinValue : int.MaxValue,
-                _ => 0,
-            })
+            .OrderByDescending(s => (s.IsExcluded, s.Name.EqualsF("pak0.pk3"))) // pak0 first, then other excludes
+            .ThenBy(s => s is DirectoryAssetSource d ? AssetDirectories.IndexOf(d.Directory) : int.MaxValue)
             .ThenByDescending(s => IOPath.GetFileNameWithoutExtension(s.RootPath))
             .ToImmutableArray();
+#pragma warning restore IDE0305 // Simplify collection initialization
     }
 
     private SourceFilter IsExcluded(FileSystemInfo item)

Pack3r.Core/Models/MapAssets.cs

@@ -7,22 +7,22 @@ public class MapAssets
     /// <summary>
     /// Shaders referenced by the .map (and possibly its mapscript etc.)
     /// </summary>
-    public required HashSet<ReadOnlyMemory<char>> Shaders { get; init; }
+    public required ResourceList Shaders { get; init; }
 
     /// <summary>
     /// Model/audio/video files referenced by the .map (and possibly its mapscript etc.)
     /// </summary>
-    public required HashSet<ReadOnlyMemory<char>> Resources { get; init; }
+    public required ResourceList Resources { get; init; }
 
     /// <summary>
     /// Models and skins that may reference other files.
     /// </summary>
-    public required HashSet<ReadOnlyMemory<char>> ReferenceResources { get; init; }
+    public required ResourceList ReferenceResources { get; init; }
 
     /// <summary>
     /// misc_models which may have remaps
     /// </summary>
-    public required Dictionary<ReadOnlyMemory<char>, List<ReferenceMiscModel>> MiscModels { get; init; }
+    public required Dictionary<Resource, List<ReferenceMiscModel>> MiscModels { get; init; }
 
     /// <summary>
     /// Whether the map has stylelights, and the q3map_mapname.shader file needs to be included.
@@ -37,7 +37,7 @@ public sealed class ReferenceMiscModel
 
     public ReferenceMiscModel(
         ReadOnlyMemory<char> model,
-        Dictionary<ReadOnlyMemory<char>, ReadOnlyMemory<char>> entitydata)
+        IEnumerable<(ReadOnlyMemory<char>, ReadOnlyMemory<char>)> entitydata)
     {
         Model = model;
 

Pack3r.Core/Models/RenamableResource.cs

@@ -4,13 +4,19 @@ namespace Pack3r.Models;
 
 public sealed class RenamableResource
 {
-    public required string AbsolutePath { get; init; }
+    public required string AbsolutePath
+    {
+        get => _absolutePath;
+        set => _absolutePath = value.NormalizePath();
+    }
+
     public required string ArchivePath
     {
         get => _archivePath;
         init => _archivePath = value.NormalizePath();
     }
 
+    private string _absolutePath = "";
     private string _archivePath = "";
 
     public Func<string, PackOptions, string>? Convert { get; init; }

Pack3r.Core/Models/Resource.cs

@@ -3,39 +3,71 @@
 
 namespace Pack3r.Models;
 
+public class IncludedFile
+{
+    public AssetSource? Source { get; }
+    public ReadOnlyMemory<char> SourcePath { get; init; }
+    public ReadOnlyMemory<char> ArchivePath { get; init; }
+    public bool SourceOnly { get; init; }
+    public string? ReferencedIn { get; init; }
+    public int? ReferencedLine { get; init; }
+
+    public IncludedFile(
+        ReadOnlyMemory<char> sourcePath,
+        ReadOnlyMemory<char> archivePath)
+    {
+        SourcePath = sourcePath;
+        ArchivePath = archivePath;
+    }
+
+    public IncludedFile(RenamableResource resource)
+    {
+        SourcePath = resource.AbsolutePath.AsMemory();
+        ArchivePath = resource.ArchivePath.AsMemory();
+    }
+
+    public IncludedFile(AssetSource source, ReadOnlyMemory<char> relativePath, Resource resource)
+    {
+        Source = source;
+        SourcePath = Path.Combine(source.RootPath, relativePath.ToString()).NormalizePath().AsMemory();
+        ArchivePath = relativePath;
+        SourceOnly = resource.SourceOnly;
+        ReferencedIn = resource.Line.Path;
+        ReferencedLine = resource.Line.Index is int i and >= 0 ? i : null;
+    }
+}
+
 /// <summary>
 /// Generic resource referenced in a map.
 /// </summary>
 /// <param name="Value">Path to the resource</param>
 /// <param name="IsShader">Whether the path is to a shader and not a file</param>
-public readonly struct Resource : IEquatable<Resource>
+public sealed class Resource(
+    ReadOnlyMemory<char> value,
+    bool isShader,
+    in Line line,
+    bool sourceOnly = false)
+    : IEquatable<Resource>
 {
-    public ReadOnlyMemory<char> Value { get; }
-    public bool IsShader { get; }
+    public static Resource Shader(string value, in Line line) => new(value.AsMemory(), true, in line);
+    public static Resource Shader(ReadOnlyMemory<char> value, in Line line) => new(value, true, in line);
 
-    public int? Line { get; }
-    public string Source { get; }
+    public static Resource File(string value, in Line line) => new(value.AsMemory(), false, in line);
+    public static Resource File(ReadOnlyMemory<char> value, in Line line) => new(value, false, in line);
 
-    public Resource(ReadOnlyMemory<char> value, bool isShader, in Line line)
-    {
-        Value = value;
-        IsShader = isShader;
+    public static Resource FromModel(
+        ReadOnlyMemory<char> value,
+        bool isShader,
+        string filePath) => new(value, isShader, new Line(filePath, -1, "", true));
 
-        Line = line.Index;
-        Source = line.Path;
-    }
+    public ReadOnlyMemory<char> Value { get; } = value;
+    public bool IsShader { get; } = isShader;
+    public Line Line { get; } = line;
+    public bool SourceOnly { get; } = sourceOnly;
 
-    public Resource(ReadOnlyMemory<char> value, bool isShader, string path)
+    public bool Equals(Resource? other)
     {
-        Value = value;
-        IsShader = isShader;
-
-        Source = path;
-    }
-
-    public bool Equals(Resource other)
-    {
-        return IsShader == other.IsShader
+        return IsShader == other?.IsShader
             && ROMCharComparer.Instance.Equals(Value, other.Value);
     }
 
@@ -49,3 +81,4 @@ public override bool Equals(object? obj)
         return obj is Resource resource && Equals(resource);
     }
 }
+

Pack3r.Core/Models/Shader.cs

@@ -17,6 +17,9 @@ public sealed class Shader(
     /// <summary>References to textures, models, videos etc</summary>
     public List<ReadOnlyMemory<char>> Resources { get; } = [];
 
+    /// <summary>References to editorimages, lightimages etc</summary>
+    public List<ReadOnlyMemory<char>> DevResources { get; } = [];
+
     /// <summary>References to other shaders</summary>
     public List<ReadOnlyMemory<char>> Shaders { get; } = [];
 

Pack3r.Core/Parsers/AseParser.cs

@@ -1,5 +1,4 @@
-using System.IO.Compression;
-using System.Text.RegularExpressions;
+using System.Text.RegularExpressions;
 using Pack3r.Extensions;
 using Pack3r.IO;
 using Pack3r.Logging;
@@ -13,9 +12,9 @@ public partial class AseParser(
 {
     public bool CanParse(ReadOnlyMemory<char> resource) => resource.EndsWithF(".ase");
 
-    public async Task<HashSet<Resource>?> Parse(IAsset asset, CancellationToken cancellationToken)
+    public async Task<ResourceList?> Parse(IAsset asset, CancellationToken cancellationToken)
     {
-        var resouces = new HashSet<Resource>();
+        ResourceList resouces = [];
 
         await foreach (var line in reader.ReadLines(asset, default, cancellationToken))
         {
@@ -29,9 +28,11 @@ await foreach (var line in reader.ReadLines(asset, default, cancellationToken))
             }
             else if (line.Value.Span.StartsWithF("*MAP_NAME"))
             {
-                if (line.Value.TryReadPastWhitespace(out var token))
+                if (line.Value.TryReadPastWhitespace(out var token) &&
+                    !token.EqualsF("\"NOSHADER\"") &&
+                    !token.EqualsF("\"textures/common/nodraw\""))
                 {
-                    resouces.Add(new Resource(token.Trim('"'), isShader: true, in line));
+                    resouces.Add(Resource.Shader(token.Trim('"'), in line));
                 }
             }
             else if (line.Value.Span.StartsWithF("*GEOMOBJECT"))

Pack3r.Core/Parsers/IReferenceParser.cs

@@ -6,7 +6,7 @@ public interface IReferenceParser
 {
     bool CanParse(ReadOnlyMemory<char> resource);
 
-    Task<HashSet<Resource>?> Parse(
+    Task<ResourceList?> Parse(
         IAsset asset,
         CancellationToken cancellationToken);
 }