diff --git a/KustoSchemaTools.Tests/Changes/ClusterGroupingTests.cs b/KustoSchemaTools.Tests/Changes/ClusterGroupingTests.cs new file mode 100644 index 0000000..a96d8a2 --- /dev/null +++ b/KustoSchemaTools.Tests/Changes/ClusterGroupingTests.cs @@ -0,0 +1,120 @@ +using KustoSchemaTools.Changes; +using KustoSchemaTools.Model; + +namespace KustoSchemaTools.Tests.Changes +{ + public class ClusterGroupingTests + { + [Fact] + public void BuildClusterFingerprint_IdenticalChanges_ProduceSameFingerprint() + { + var changes1 = CreateSampleChanges("## Table1\nSome diff"); + var changes2 = CreateSampleChanges("## Table1\nSome diff"); + var comments = new List(); + + var fp1 = KustoSchemaHandler.BuildClusterFingerprint(changes1, comments, true); + var fp2 = KustoSchemaHandler.BuildClusterFingerprint(changes2, comments, true); + + Assert.Equal(fp1, fp2); + } + + [Fact] + public void BuildClusterFingerprint_DifferentMarkdown_ProduceDifferentFingerprints() + { + var changes1 = CreateSampleChanges("## Table1\nDiff A"); + var changes2 = CreateSampleChanges("## Table1\nDiff B"); + var comments = new List(); + + var fp1 = KustoSchemaHandler.BuildClusterFingerprint(changes1, comments, true); + var fp2 = KustoSchemaHandler.BuildClusterFingerprint(changes2, comments, true); + + Assert.NotEqual(fp1, fp2); + } + + [Fact] + public void BuildClusterFingerprint_DifferentValidity_ProduceDifferentFingerprints() + { + var changes = CreateSampleChanges("## Table1\nSame diff"); + var comments = new List(); + + var fp1 = KustoSchemaHandler.BuildClusterFingerprint(changes, comments, true); + var fp2 = KustoSchemaHandler.BuildClusterFingerprint(changes, comments, false); + + Assert.NotEqual(fp1, fp2); + } + + [Fact] + public void BuildClusterFingerprint_DifferentComments_ProduceDifferentFingerprints() + { + var changes = CreateSampleChanges("## Table1\nSame diff"); + var comments1 = new List + { + new Comment { Kind = CommentKind.Warning, Text = "Warning 1", FailsRollout = false } + }; + var comments2 = new List + { + new Comment { Kind = CommentKind.Caution, Text = "Caution 1", FailsRollout = true } + }; + + var fp1 = KustoSchemaHandler.BuildClusterFingerprint(changes, comments1, true); + var fp2 = KustoSchemaHandler.BuildClusterFingerprint(changes, comments2, true); + + Assert.NotEqual(fp1, fp2); + } + + [Fact] + public void BuildClusterFingerprint_EmptyChanges_ProduceSameFingerprint() + { + var changes = new List(); + var comments = new List(); + + var fp1 = KustoSchemaHandler.BuildClusterFingerprint(changes, comments, true); + var fp2 = KustoSchemaHandler.BuildClusterFingerprint(changes, comments, true); + + Assert.Equal(fp1, fp2); + } + + [Fact] + public void BuildClusterFingerprint_CommentsInDifferentOrder_ProduceSameFingerprint() + { + var changes = CreateSampleChanges("## Table1\nSame diff"); + var comments1 = new List + { + new Comment { Kind = CommentKind.Warning, Text = "A", FailsRollout = false }, + new Comment { Kind = CommentKind.Note, Text = "B", FailsRollout = false } + }; + var comments2 = new List + { + new Comment { Kind = CommentKind.Note, Text = "B", FailsRollout = false }, + new Comment { Kind = CommentKind.Warning, Text = "A", FailsRollout = false } + }; + + var fp1 = KustoSchemaHandler.BuildClusterFingerprint(changes, comments1, true); + var fp2 = KustoSchemaHandler.BuildClusterFingerprint(changes, comments2, true); + + Assert.Equal(fp1, fp2); + } + + private static List CreateSampleChanges(string markdown) + { + return new List + { + new FakeChange(markdown) + }; + } + + private class FakeChange : IChange + { + public FakeChange(string markdown) + { + Markdown = markdown; + } + + public string EntityType => "Test"; + public string Entity => "TestEntity"; + public List Scripts => new List(); + public string Markdown { get; } + public Comment Comment { get; set; } + } + } +} diff --git a/KustoSchemaTools.Tests/Changes/ColumnDiffHelperTests.cs b/KustoSchemaTools.Tests/Changes/ColumnDiffHelperTests.cs new file mode 100644 index 0000000..0f6f1c7 --- /dev/null +++ b/KustoSchemaTools.Tests/Changes/ColumnDiffHelperTests.cs @@ -0,0 +1,185 @@ +using KustoSchemaTools.Changes; + +namespace KustoSchemaTools.Tests.Changes +{ + public class ColumnDiffHelperTests + { + [Fact] + public void BuildColumnDiff_WithAddedColumns_ShowsAdditions() + { + var oldColumns = new Dictionary + { + { "id", "string" }, + { "timestamp", "datetime" } + }; + var newColumns = new Dictionary + { + { "id", "string" }, + { "timestamp", "datetime" }, + { "new_col", "long" } + }; + + var result = ColumnDiffHelper.BuildColumnDiff(oldColumns, newColumns); + + Assert.NotNull(result); + Assert.Contains("+ new_col: long", result); + Assert.DoesNotContain("id", result); + Assert.DoesNotContain("timestamp", result); + } + + [Fact] + public void BuildColumnDiff_WithTypeChange_ShowsTypeChange() + { + var oldColumns = new Dictionary + { + { "id", "string" }, + { "count", "int" } + }; + var newColumns = new Dictionary + { + { "id", "string" }, + { "count", "long" } + }; + + var result = ColumnDiffHelper.BuildColumnDiff(oldColumns, newColumns); + + Assert.NotNull(result); + Assert.Contains("! count: int → long", result); + Assert.DoesNotContain("id", result); + } + + [Fact] + public void BuildColumnDiff_WithRemovedColumns_ShowsInformationalNote() + { + var oldColumns = new Dictionary + { + { "id", "string" }, + { "old_col", "string" }, + { "timestamp", "datetime" } + }; + var newColumns = new Dictionary + { + { "id", "string" }, + { "timestamp", "datetime" } + }; + + var result = ColumnDiffHelper.BuildColumnDiff(oldColumns, newColumns); + + Assert.NotNull(result); + Assert.Contains("old_col: string (live only)", result); + Assert.Contains(".create-merge", result); + Assert.DoesNotContain("+ ", result); + } + + [Fact] + public void BuildColumnDiff_WithNoChanges_ReturnsNull() + { + var oldColumns = new Dictionary + { + { "id", "string" }, + { "timestamp", "datetime" } + }; + var newColumns = new Dictionary + { + { "id", "string" }, + { "timestamp", "datetime" } + }; + + var result = ColumnDiffHelper.BuildColumnDiff(oldColumns, newColumns); + + Assert.Null(result); + } + + [Fact] + public void BuildColumnDiff_WithNullOldColumns_TreatsAsNewTable() + { + var newColumns = new Dictionary + { + { "id", "string" }, + { "timestamp", "datetime" } + }; + + var result = ColumnDiffHelper.BuildColumnDiff(null, newColumns); + + Assert.NotNull(result); + Assert.Contains("+ id: string", result); + Assert.Contains("+ timestamp: datetime", result); + } + + [Fact] + public void BuildColumnDiff_WithNullNewColumns_ReturnsNull() + { + var oldColumns = new Dictionary + { + { "id", "string" } + }; + + var result = ColumnDiffHelper.BuildColumnDiff(oldColumns, null); + + Assert.Null(result); + } + + [Fact] + public void BuildColumnDiff_WithMixedChanges_ShowsAllCategories() + { + var oldColumns = new Dictionary + { + { "id", "string" }, + { "count", "int" }, + { "removed_col", "string" } + }; + var newColumns = new Dictionary + { + { "id", "string" }, + { "count", "long" }, + { "new_col", "datetime" } + }; + + var result = ColumnDiffHelper.BuildColumnDiff(oldColumns, newColumns); + + Assert.NotNull(result); + Assert.Contains("+ new_col: datetime", result); + Assert.Contains("! count: int → long", result); + Assert.Contains("removed_col: string (live only)", result); + } + + [Fact] + public void BuildColumnDiff_CaseInsensitiveTypeComparison_IgnoresCaseDifference() + { + var oldColumns = new Dictionary + { + { "id", "String" } + }; + var newColumns = new Dictionary + { + { "id", "string" } + }; + + var result = ColumnDiffHelper.BuildColumnDiff(oldColumns, newColumns); + + Assert.Null(result); + } + + [Fact] + public void BuildColumnDiff_ReorderOnly_ReturnsNull() + { + var oldColumns = new Dictionary + { + { "id", "string" }, + { "timestamp", "datetime" }, + { "count", "long" } + }; + // Same columns, different insertion order in the dictionary + var newColumns = new Dictionary + { + { "count", "long" }, + { "id", "string" }, + { "timestamp", "datetime" } + }; + + var result = ColumnDiffHelper.BuildColumnDiff(oldColumns, newColumns); + + Assert.Null(result); + } + } +} diff --git a/KustoSchemaTools/Changes/ColumnDiffHelper.cs b/KustoSchemaTools/Changes/ColumnDiffHelper.cs new file mode 100644 index 0000000..aff7c55 --- /dev/null +++ b/KustoSchemaTools/Changes/ColumnDiffHelper.cs @@ -0,0 +1,74 @@ +using KustoSchemaTools.Model; +using System.Text; + +namespace KustoSchemaTools.Changes +{ + /// + /// Generates a human-readable column-level diff for table schema changes. + /// Instead of showing full .create-merge table lines, shows only the columns + /// that were added or had their type changed. + /// + public static class ColumnDiffHelper + { + /// + /// Builds a column-level diff between two sets of columns. + /// + /// Columns from the live cluster state (may be null for new tables). + /// Columns from the desired YAML state. + /// + /// A formatted diff string showing added columns and type changes, + /// or null if there are no meaningful column differences. + /// + public static string? BuildColumnDiff(Dictionary? oldColumns, Dictionary? newColumns) + { + if (newColumns == null) return null; + oldColumns ??= new Dictionary(); + + var added = newColumns + .Where(c => !oldColumns.ContainsKey(c.Key)) + .ToList(); + + var typeChanged = newColumns + .Where(c => oldColumns.ContainsKey(c.Key) && !string.Equals(oldColumns[c.Key], c.Value, StringComparison.OrdinalIgnoreCase)) + .Select(c => new { Name = c.Key, OldType = oldColumns[c.Key], NewType = c.Value }) + .ToList(); + + var removedFromYaml = oldColumns + .Where(c => !newColumns.ContainsKey(c.Key)) + .ToList(); + + if (!added.Any() && !typeChanged.Any() && !removedFromYaml.Any()) + return null; + + var sb = new StringBuilder(); + sb.AppendLine("```diff"); + + foreach (var col in added) + { + sb.AppendLine($"+ {col.Key}: {col.Value}"); + } + + foreach (var col in typeChanged) + { + sb.AppendLine($"! {col.Name}: {col.OldType} → {col.NewType}"); + } + + if (removedFromYaml.Any()) + { + sb.AppendLine("```"); + sb.AppendLine(); + sb.AppendLine("> **Note**: The following columns exist in the live cluster but not in YAML."); + sb.AppendLine("> `.create-merge` does not remove columns — they will remain on the table."); + sb.AppendLine(); + sb.AppendLine("```"); + foreach (var col in removedFromYaml) + { + sb.AppendLine($" {col.Key}: {col.Value} (live only)"); + } + } + + sb.AppendLine("```"); + return sb.ToString(); + } + } +} diff --git a/KustoSchemaTools/Changes/ScriptCompareChange.cs b/KustoSchemaTools/Changes/ScriptCompareChange.cs index 8d63465..0df6e08 100644 --- a/KustoSchemaTools/Changes/ScriptCompareChange.cs +++ b/KustoSchemaTools/Changes/ScriptCompareChange.cs @@ -77,37 +77,10 @@ private void Init() sb.AppendLine($""); if (before != null) { - foreach(var c in new [] { new { Change = ChangeType.Deleted, Prefix = "-" }, new { Change = ChangeType.Inserted, Prefix = "+" } }) + var columnDiffRendered = TryRenderColumnDiff(change.Kind, sb); + if (!columnDiffRendered) { - var changeType = c.Change; - if (diff.Lines.Any(itm => itm.Type == changeType)) - { - sb.AppendLine(""); - sb.AppendLine($" {c.Change}:"); - sb.AppendLine($" \n\n```diff "); - - var relevantLines = diff.Lines.Where(itm => itm.Type == ChangeType.Unchanged || itm.Type == changeType).OrderBy(itm => itm.Position).ToList(); - int last = 0; - for (int i = 0; i < relevantLines.Count; i++) - { - var b = i - 1 > 0 ? relevantLines[i - 1] : null; - var current = relevantLines[i]; - var n = i + 1 < relevantLines.Count ? relevantLines[i + 1] : null; - - if (current.Type == changeType || b?.Type == changeType || n?.Type == changeType) - { - if(i-last > 1) - { - sb.AppendLine(); - } - - var p = current.Type == changeType ? c.Prefix : " "; - sb.AppendLine($"{p}{i}:\t{current.Text}"); - last = i; - } - } - sb.AppendLine("```\n\n"); - } + RenderLineDiff(diff, sb); } } sb.AppendLine(""); @@ -129,6 +102,64 @@ private void Init() sb.AppendLine(""); Markdown = sb.ToString(); } + + /// + /// Attempts to render a column-level diff for CreateMergeTable changes. + /// Returns true if a column diff was rendered, false to fall back to line diff. + /// + private bool TryRenderColumnDiff(string kind, StringBuilder sb) + { + if (kind != "CreateMergeTable") return false; + + var oldTable = From as Table; + var newTable = To as Table; + if (newTable?.Columns == null) return false; + + var columnDiff = ColumnDiffHelper.BuildColumnDiff(oldTable?.Columns, newTable.Columns); + if (columnDiff == null) return false; + + sb.AppendLine(""); + sb.AppendLine(" Column changes:"); + sb.AppendLine($" \n\n{columnDiff}\n\n"); + sb.AppendLine(""); + return true; + } + + private static void RenderLineDiff(DiffPaneModel diff, StringBuilder sb) + { + foreach (var c in new[] { new { Change = ChangeType.Deleted, Prefix = "-" }, new { Change = ChangeType.Inserted, Prefix = "+" } }) + { + var changeType = c.Change; + if (diff.Lines.Any(itm => itm.Type == changeType)) + { + sb.AppendLine(""); + sb.AppendLine($" {c.Change}:"); + sb.AppendLine($" \n\n```diff "); + + var relevantLines = diff.Lines.Where(itm => itm.Type == ChangeType.Unchanged || itm.Type == changeType).OrderBy(itm => itm.Position).ToList(); + int last = 0; + for (int i = 0; i < relevantLines.Count; i++) + { + var b = i - 1 > 0 ? relevantLines[i - 1] : null; + var current = relevantLines[i]; + var n = i + 1 < relevantLines.Count ? relevantLines[i + 1] : null; + + if (current.Type == changeType || b?.Type == changeType || n?.Type == changeType) + { + if (i - last > 1) + { + sb.AppendLine(); + } + + var p = current.Type == changeType ? c.Prefix : " "; + sb.AppendLine($"{p}{i}:\t{current.Text}"); + last = i; + } + } + sb.AppendLine("```\n\n"); + } + } + } } } diff --git a/KustoSchemaTools/KustoSchemaHandler.cs b/KustoSchemaTools/KustoSchemaHandler.cs index c5be0d9..3141c73 100644 --- a/KustoSchemaTools/KustoSchemaHandler.cs +++ b/KustoSchemaTools/KustoSchemaHandler.cs @@ -147,49 +147,73 @@ private async Task BuildDiffComputationResult(string path var sb = new StringBuilder(); bool isValid = true; + // Compute per-cluster metadata first (for logging and validity) + var clusterRenderModels = new List(); foreach (var clusterDiff in diffData.ClusterDiffs) { + var changes = clusterDiff.Changes; + var comments = changes.Select(change => change.Comment).OfType().ToList(); + var clusterValid = IsDiffValid(changes); + isValid &= clusterValid; + + var fingerprint = BuildClusterFingerprint(changes, comments, clusterValid); + + clusterRenderModels.Add(new ClusterRenderModel(clusterDiff, changes, comments, clusterValid, fingerprint)); + if (logDetails) { Log.LogInformation($"Generating diff markdown for {Path.Join(path, databaseName)} => {clusterDiff.Cluster.Name}/{databaseName}"); + var scriptSb = new StringBuilder(); + foreach (var script in changes.SelectMany(itm => itm.Scripts).Where(itm => itm.IsValid is true).OrderBy(itm => itm.Script.Order)) + { + scriptSb.AppendLine(script.Script.Text); + } + Log.LogInformation($"Following scripts will be applied:\n{scriptSb}"); } + } - var changes = clusterDiff.Changes; - var comments = changes.Select(change => change.Comment).OfType().ToList(); - var clusterValid = IsDiffValid(changes); - isValid &= clusterValid; + // Group clusters with identical diffs + var groups = clusterRenderModels.GroupBy(m => m.Fingerprint).ToList(); - sb.AppendLine($"# {clusterDiff.Cluster.Name}/{databaseName} ({clusterDiff.Cluster.Url})"); + foreach (var group in groups) + { + var clusters = group.ToList(); + var representative = clusters.First(); + + // Build combined header + var clusterHeaders = clusters.Select(c => $"{c.Context.Cluster.Name}/{databaseName} ({c.Context.Cluster.Url})"); + if (clusters.Count == 1) + { + sb.AppendLine($"# {clusterHeaders.First()}"); + } + else + { + sb.AppendLine($"# {clusters.Count} clusters with identical changes"); + foreach (var header in clusterHeaders) + { + sb.AppendLine($"- {header}"); + } + sb.AppendLine(); + } - foreach (var comment in comments) + foreach (var comment in representative.Comments) { sb.AppendLine($"> [!{comment.Kind.ToString().ToUpper()}]"); sb.AppendLine($"> {comment.Text}"); sb.AppendLine(); } - if (changes.Count == 0) + if (representative.Changes.Count == 0) { sb.AppendLine("No changes detected"); } - foreach (var change in changes) + foreach (var change in representative.Changes) { sb.AppendLine(change.Markdown); sb.AppendLine(); sb.AppendLine(); } - - if (logDetails) - { - var scriptSb = new StringBuilder(); - foreach (var script in changes.SelectMany(itm => itm.Scripts).Where(itm => itm.IsValid is true).OrderBy(itm => itm.Script.Order)) - { - scriptSb.AppendLine(script.Script.Text); - } - - Log.LogInformation($"Following scripts will be applied:\n{scriptSb}"); - } } foreach (var followerDiff in diffData.FollowerDiffs) @@ -214,6 +238,46 @@ private async Task BuildDiffComputationResult(string path return (sb.ToString(), isValid); } + /// + /// Builds a canonical fingerprint for a cluster's diff output. + /// Clusters with the same fingerprint will be grouped together in the markdown. + /// + public static string BuildClusterFingerprint(List changes, List comments, bool isValid) + { + var sb = new StringBuilder(); + sb.Append($"valid:{isValid};"); + + foreach (var comment in comments.OrderBy(c => c.Kind).ThenBy(c => c.Text)) + { + sb.Append($"comment:{comment.Kind}:{comment.FailsRollout}:{comment.Text};"); + } + + foreach (var change in changes) + { + sb.Append($"change:{change.Markdown};"); + } + + return sb.ToString(); + } + + private sealed class ClusterRenderModel + { + public ClusterRenderModel(ClusterDiffContext context, List changes, List comments, bool isValid, string fingerprint) + { + Context = context; + Changes = changes; + Comments = comments; + IsValid = isValid; + Fingerprint = fingerprint; + } + + public ClusterDiffContext Context { get; } + public List Changes { get; } + public List Comments { get; } + public bool IsValid { get; } + public string Fingerprint { get; } + } + private StructuredDiff ConvertToStructuredDiff(string clusterName, string clusterUrl, string databaseName, List changes) { var structuredChanges = changes.Select(change => change.ToStructuredChange()).ToList();