fix: attempt hungarian solver on dropped neural

Author: Spelkington

examples/advanced/DroppedNeuralNet/Data/_01_Raw/Catalog.Raw.cs

@@ -36,10 +36,16 @@ public partial class Catalog
 
   /// <summary>
   /// Raw layer blobs loaded from the pieces directory.
-  /// Produced by the load_pieces Python step; held in memory throughout the pipeline.
-  /// The pieces directory path is passed as configuration — Flowthru is not responsible
-  /// for managing the individual .pth files.
+  /// Produced by the load_pieces Python step; persisted as JSON (base64-encoded Data field)
+  /// so downstream flows (Exploration, Validation, Solver) can run independently without
+  /// re-executing DataPrep from scratch.
   /// </summary>
   public IItem<IEnumerable<PieceBlob>> Pieces =>
-    CreateItem(() => ItemFactory.Enumerable.Memory<PieceBlob>(label: "Pieces"));
+    CreateItem(
+      () =>
+        ItemFactory.Enumerable.Json<PieceBlob>(
+          label: "Pieces",
+          filePath: $"{_basePath}/_01_Raw/Datasets/pieces.json"
+        )
+    );
 }

examples/advanced/DroppedNeuralNet/Data/_01_Raw/Datasets/pieces.json

@@ -0,0 +1,34 @@
+using DroppedNeuralNet.Data._04_Analysis.Schemas;
+using Flowthru.Core.Data;
+
+namespace DroppedNeuralNet.Data;
+
+public partial class Catalog
+{
+  /// <summary>
+  /// Frobenius norms of the weight product ||out.W @ inp.W||_F for every legal (inp, out) pairing.
+  /// Produced by the compute_pairing_scores Python step; persisted as JSON for inspection.
+  /// </summary>
+  public IItem<IEnumerable<PairingScore>> PairingScores =>
+    CreateItem(
+      () =>
+        ItemFactory.Enumerable.Json<PairingScore>(
+          label: "PairingScores",
+          filePath: $"{_basePath}/_04_Analysis/Datasets/pairing_scores.json"
+        )
+    );
+
+  /// <summary>
+  /// Optimal (inp, out) Block pairings selected by the Hungarian algorithm.
+  /// Minimises total ProductNorm across all 48 Block assignments.
+  /// Persisted as JSON; consumed by the rank_orderings step.
+  /// </summary>
+  public IItem<IEnumerable<BlockAssignment>> BlockAssignments =>
+    CreateItem(
+      () =>
+        ItemFactory.Enumerable.Json<BlockAssignment>(
+          label: "BlockAssignments",
+          filePath: $"{_basePath}/_04_Analysis/Datasets/block_assignments.json"
+        )
+    );
+}

examples/advanced/DroppedNeuralNet/Data/_04_Analysis/Catalog.Analysis.cs

@@ -0,0 +1,20 @@
+using Flowthru.Core.Abstractions;
+
+namespace DroppedNeuralNet.Data._04_Analysis.Schemas;
+
+/// <summary>
+/// A single Block pairing selected by the Hungarian algorithm: the globally optimal
+/// assignment of inp pieces to out pieces under the minimum total ProductNorm objective.
+/// </summary>
+[FlowthruSchema]
+public partial record BlockAssignment
+{
+  /// <summary>Sequential Block position assigned by the Hungarian solver (0–47). Not the execution order.</summary>
+  public int BlockIndex { get; init; }
+
+  public int InpPieceIndex { get; init; }
+  public int OutPieceIndex { get; init; }
+
+  /// <summary>ProductNorm score for this pairing — the cost the solver minimized.</summary>
+  public float AssignmentScore { get; init; }
+}

examples/advanced/DroppedNeuralNet/Data/_04_Analysis/Schemas/BlockAssignment.cs

@@ -0,0 +1,18 @@
+using Flowthru.Core.Abstractions;
+
+namespace DroppedNeuralNet.Data._04_Analysis.Schemas;
+
+/// <summary>
+/// The Frobenius norm of the product ||out.W @ inp.W||_F for a candidate Block pairing.
+/// Lower values indicate that the out layer is a better inverse of the inp layer,
+/// which is the structural signal left in the weights by the residual training objective.
+/// </summary>
+[FlowthruSchema]
+public partial record PairingScore
+{
+  public int InpPieceIndex { get; init; }
+  public int OutPieceIndex { get; init; }
+
+  /// <summary>||W_out @ W_inp||_F. Lower = stronger residual coupling between these two layers.</summary>
+  public float ProductNorm { get; init; }
+}

examples/advanced/DroppedNeuralNet/Data/_04_Analysis/Schemas/PairingScore.cs

@@ -0,0 +1,21 @@
+using DroppedNeuralNet.Data._05_Candidates.Schemas;
+using Flowthru.Core.Data;
+
+namespace DroppedNeuralNet.Data;
+
+public partial class Catalog
+{
+  /// <summary>
+  /// Ranked candidate execution orderings produced by the rank_orderings step.
+  /// Each row is a JSON-encoded int[97] permutation. Persisted as JSON so the
+  /// Solver flow can be run independently after Exploration completes.
+  /// </summary>
+  public IItem<IEnumerable<CandidatePermutation>> CandidatePermutations =>
+    CreateItem(
+      () =>
+        ItemFactory.Enumerable.Json<CandidatePermutation>(
+          label: "CandidatePermutations",
+          filePath: $"{_basePath}/_05_Candidates/Datasets/candidate_permutations.json"
+        )
+    );
+}

examples/advanced/DroppedNeuralNet/Data/_05_Candidates/Catalog.Candidates.cs

@@ -0,0 +1,22 @@
+using Flowthru.Core.Abstractions;
+
+namespace DroppedNeuralNet.Data._05_Candidates.Schemas;
+
+/// <summary>
+/// A single candidate execution ordering for the 97 network pieces.
+/// The permutation is JSON-encoded so it can cross the Arrow boundary as a flat string
+/// and be decoded by Python steps without a nested-schema storage adapter.
+/// </summary>
+[FlowthruSchema]
+public partial record CandidatePermutation
+{
+  public int CandidateIndex { get; init; }
+
+  /// <summary>
+  /// JSON-encoded int array of length 97.
+  /// Positions 2k and 2k+1 are the inp/out pieces for Block k (k = 0..47).
+  /// Position 96 is the LastLayer piece index.
+  /// Example: "[12, 45, 3, 67, ...]"
+  /// </summary>
+  public string Permutation { get; init; } = "[]";
+}

examples/advanced/DroppedNeuralNet/Data/_05_Candidates/Schemas/CandidatePermutation.cs

@@ -0,0 +1,22 @@
+using DroppedNeuralNet.Data._08_Reporting.Schemas;
+using Flowthru.Core.Data;
+
+namespace DroppedNeuralNet.Data;
+
+public partial class Catalog
+{
+  /// <summary>
+  /// Diagnostic measurements from the Validation flow.
+  /// Each row is a (Category, Metric, Value, Notes) tuple covering pairing signal
+  /// quality, fixed-ordering baseline error, and per-candidate forward-pass errors.
+  /// Persisted as JSON so reports survive process exit and can be inspected offline.
+  /// </summary>
+  public IItem<IEnumerable<DiagnosticEntry>> Diagnostics =>
+    CreateItem(
+      () =>
+        ItemFactory.Enumerable.Json<DiagnosticEntry>(
+          label: "Diagnostics",
+          filePath: $"{_basePath}/_08_Reporting/Datasets/diagnostics.json"
+        )
+    );
+}

examples/advanced/DroppedNeuralNet/Data/_08_Reporting/Catalog.Reporting.cs

@@ -0,0 +1,27 @@
+using Flowthru.Core.Abstractions;
+
+namespace DroppedNeuralNet.Data._08_Reporting.Schemas;
+
+/// <summary>
+/// A single diagnostic measurement emitted by the Validation flow.
+///
+/// Rows are grouped by <see cref="Category"/> (e.g. "FixedOrdering", "Candidate_3",
+/// "PairingSignal") and keyed by <see cref="Metric"/> within each group. This flat
+/// schema lets a single step emit heterogeneous signal types without requiring a
+/// separate schema per check.
+/// </summary>
+[FlowthruSchema]
+public partial record DiagnosticEntry
+{
+  /// <summary>Logical grouping, e.g. "FixedOrdering", "Candidate_7", "PairingSignal".</summary>
+  public string Category { get; init; } = "";
+
+  /// <summary>Name of the measured quantity, e.g. "MaxErr", "MeanErr", "ScoreStd".</summary>
+  public string Metric { get; init; } = "";
+
+  /// <summary>Numeric value of the measurement.</summary>
+  public float Value { get; init; }
+
+  /// <summary>Optional human-readable context emitted alongside the value.</summary>
+  public string Notes { get; init; } = "";
+}

examples/advanced/DroppedNeuralNet/Data/_08_Reporting/Schemas/DiagnosticEntry.cs

@@ -0,0 +1,88 @@
+using DroppedNeuralNet.Data;
+using DroppedNeuralNet.Data._01_Raw.Schemas;
+using DroppedNeuralNet.Data._02_Intermediate.Schemas;
+using DroppedNeuralNet.Data._03_Primary.Schemas;
+using DroppedNeuralNet.Data._04_Analysis.Schemas;
+using DroppedNeuralNet.Data._05_Candidates.Schemas;
+using DroppedNeuralNet.Flows.Exploration.Steps;
+using Flowthru.Core.Flows;
+using Flowthru.Extensions.Python.Execution;
+using Flowthru.Extensions.Python.Steps;
+
+namespace DroppedNeuralNet.Flows.Exploration;
+
+/// <summary>
+/// Analytical pipeline that narrows the search space before the Solver validates permutations.
+///
+/// Step 1 (C#)    — FindLegalPairings:
+///   Sieves PieceMetadata to enumerate every structurally valid (inp, out) Block candidate.
+///   Pure dimension arithmetic; no blobs.
+///
+/// Step 2 (Python) — compute_pairing_scores:
+///   For every legal (inp, out) candidate, computes ||W_out @ W_inp||_F.
+///   Lower scores indicate residual coupling — the signal left by training.
+///
+/// Step 3 (Python) — run_hungarian:
+///   Applies the Hungarian algorithm to the 48×48 score matrix.
+///   Produces the globally optimal assignment of inp ↔ out pieces in O(n³).
+///
+/// Step 4 (Python) — rank_orderings:
+///   Uses activation chaining on historical data to score Block execution orderings.
+///   Emits a small ranked set of candidate permutations for the Solver to validate.
+/// </summary>
+public static class ExplorationFlow
+{
+  public static Flow Create(Catalog catalog, IPythonExecutor executor)
+  {
+    return FlowBuilder.CreateFlow(pipeline =>
+    {
+      pipeline.AddStep(
+        label: "FindLegalPairings",
+        description: "Sieve PieceMetadata to all dimension-valid (inp, out) Block candidates (C#).",
+        transform: FindLegalPairingsStep.Create(),
+        input: catalog.PieceMetadata,
+        output: catalog.LegalPairings
+      );
+
+      pipeline.AddPythonStep<
+        IEnumerable<PieceMetadata>,
+        IEnumerable<PieceBlob>,
+        IEnumerable<BlockCandidate>,
+        IEnumerable<PairingScore>
+      >(
+        label: "ComputePairingScores",
+        description: "Compute ||W_out @ W_inp||_F for every legal pairing. Lower = stronger residual coupling (Python).",
+        module: "Flows.Exploration.Steps.compute_pairing_scores",
+        function: "compute_pairing_scores",
+        input: (catalog.PieceMetadata, catalog.Pieces, catalog.LegalPairings),
+        output: catalog.PairingScores,
+        executor: executor
+      );
+
+      pipeline.AddPythonStep<IEnumerable<PairingScore>, IEnumerable<BlockAssignment>>(
+        label: "RunHungarian",
+        description: "Globally optimal inp↔out assignment via Hungarian algorithm on the 48×48 score matrix (Python).",
+        module: "Flows.Exploration.Steps.run_hungarian",
+        function: "run_hungarian",
+        input: catalog.PairingScores,
+        output: catalog.BlockAssignments,
+        executor: executor
+      );
+
+      pipeline.AddPythonStep<
+        IEnumerable<BlockAssignment>,
+        IEnumerable<PieceBlob>,
+        IEnumerable<MeasurementSchema>,
+        IEnumerable<CandidatePermutation>
+      >(
+        label: "RankOrderings",
+        description: "Score Block execution orderings via activation chaining; emit top-N candidate permutations (Python).",
+        module: "Flows.Exploration.Steps.rank_orderings",
+        function: "rank_orderings",
+        input: (catalog.BlockAssignments, catalog.Pieces, catalog.HistoricalData),
+        output: catalog.CandidatePermutations,
+        executor: executor
+      );
+    });
+  }
+}