From 5eeba68c80546cff1691cc05dbff3398d748dcbe Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sat, 25 Jul 2026 07:14:17 +0200 Subject: [PATCH 1/8] Add representation-preserving YAML merging Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../private/Assert-YamlMergeGraph.ps1 | 125 ++++ src/functions/private/Copy-YamlMergeNode.ps1 | 167 +++++ .../private/Get-YamlMergeFingerprint.ps1 | 30 + src/functions/private/Get-YamlMergeNode.ps1 | 18 + .../private/Get-YamlMergeNodeTag.ps1 | 20 + src/functions/private/Get-YamlMergePath.ps1 | 29 + .../private/Merge-YamlRepresentationNode.ps1 | 201 ++++++ .../private/New-YamlMergeException.ps1 | 30 + .../private/Test-YamlMergeNodeEqual.ps1 | 180 ++++++ src/functions/public/Merge-Yaml.ps1 | 258 ++++++++ tests/Merge-Yaml.Tests.ps1 | 594 ++++++++++++++++++ 11 files changed, 1652 insertions(+) create mode 100644 src/functions/private/Assert-YamlMergeGraph.ps1 create mode 100644 src/functions/private/Copy-YamlMergeNode.ps1 create mode 100644 src/functions/private/Get-YamlMergeFingerprint.ps1 create mode 100644 src/functions/private/Get-YamlMergeNode.ps1 create mode 100644 src/functions/private/Get-YamlMergeNodeTag.ps1 create mode 100644 src/functions/private/Get-YamlMergePath.ps1 create mode 100644 src/functions/private/Merge-YamlRepresentationNode.ps1 create mode 100644 src/functions/private/New-YamlMergeException.ps1 create mode 100644 src/functions/private/Test-YamlMergeNodeEqual.ps1 create mode 100644 src/functions/public/Merge-Yaml.ps1 create mode 100644 tests/Merge-Yaml.Tests.ps1 diff --git a/src/functions/private/Assert-YamlMergeGraph.ps1 b/src/functions/private/Assert-YamlMergeGraph.ps1 new file mode 100644 index 0000000..9979ff2 --- /dev/null +++ b/src/functions/private/Assert-YamlMergeGraph.ps1 @@ -0,0 +1,125 @@ +function Assert-YamlMergeGraph { + <# + .SYNOPSIS + Validates the merged representation graph and its resource budgets. + #> + [CmdletBinding()] + param ( + [Parameter(Mandatory)] + [AllowEmptyCollection()] + [object[]] $Documents, + + [Parameter(Mandatory)] + [int] $Depth, + + [Parameter(Mandatory)] + [int] $MaxNodes, + + [Parameter(Mandatory)] + [int] $MaxAliases, + + [Parameter(Mandatory)] + [int] $MaxScalarLength, + + [Parameter(Mandatory)] + [int] $MaxTagLength, + + [Parameter(Mandatory)] + [int] $MaxTotalTagLength + ) + + $visited = [System.Collections.Generic.HashSet[int]]::new() + $pending = [System.Collections.Generic.Stack[object]]::new() + foreach ($document in $Documents) { + $pending.Push([pscustomobject]@{ Node = $document; Depth = 1 }) + } + $nodeCount = 0 + $aliasCount = 0 + $totalTagLength = 0L + + while ($pending.Count -gt 0) { + $item = $pending.Pop() + $node = $item.Node + if (-not $visited.Add($node.Id)) { + continue + } + $nodeCount++ + if ($nodeCount -gt $MaxNodes) { + throw (New-YamlMergeException -Node $node -ErrorId 'YamlMergeNodeLimitExceeded' -Message ( + "The merged YAML graph exceeds the configured limit of $MaxNodes nodes." + )) + } + if ($item.Depth -gt $Depth) { + throw (New-YamlMergeException -Node $node -ErrorId 'YamlMergeDepthLimitExceeded' -Message ( + "The merged YAML graph exceeds the configured depth of $Depth." + )) + } + + if ($node.Kind -eq 'Alias') { + $aliasCount++ + if ($aliasCount -gt $MaxAliases) { + throw (New-YamlMergeException -Node $node -ErrorId 'YamlMergeAliasLimitExceeded' -Message ( + "The merged YAML graph exceeds the configured limit of $MaxAliases aliases." + )) + } + $pending.Push([pscustomobject]@{ Node = $node.Target; Depth = $item.Depth }) + continue + } + + $tagLength = ([string] $node.Tag).Length + if ($tagLength -gt $MaxTagLength) { + throw (New-YamlMergeException -Node $node -ErrorId 'YamlMergeTagLimitExceeded' -Message ( + "A tag in the merged YAML graph exceeds the configured limit of $MaxTagLength characters." + )) + } + $totalTagLength += $tagLength + if ($totalTagLength -gt $MaxTotalTagLength) { + throw (New-YamlMergeException -Node $node -ErrorId 'YamlMergeTagLimitExceeded' -Message ( + "The merged YAML graph exceeds the configured cumulative tag limit of " + + "$MaxTotalTagLength characters." + )) + } + + if ($node.Kind -eq 'Scalar') { + if ((Get-YamlRuneCount -Text ([string] $node.Value)) -gt $MaxScalarLength) { + throw (New-YamlMergeException -Node $node ` + -ErrorId 'YamlMergeScalarLimitExceeded' -Message ( + "A scalar in the merged YAML graph exceeds the configured limit of " + + "$MaxScalarLength characters." + )) + } + continue + } + if ($node.Kind -eq 'Sequence') { + for ($index = $node.Items.Count - 1; $index -ge 0; $index--) { + $pending.Push([pscustomobject]@{ + Node = $node.Items[$index] + Depth = $item.Depth + 1 + }) + } + continue + } + for ($index = $node.Entries.Count - 1; $index -ge 0; $index--) { + $pending.Push([pscustomobject]@{ + Node = $node.Entries[$index].Value + Depth = $item.Depth + 1 + }) + $pending.Push([pscustomobject]@{ + Node = $node.Entries[$index].Key + Depth = $item.Depth + 1 + }) + } + } + + $fingerprintCache = [System.Collections.Generic.Dictionary[int, string]]::new() + $fingerprintHasher = [System.Security.Cryptography.SHA256]::Create() + try { + foreach ($document in $Documents) { + Test-YamlNodeGraph -Node $document ` + -Visited ([System.Collections.Generic.HashSet[int]]::new()) ` + -FingerprintCache $fingerprintCache -FingerprintHasher $fingerprintHasher + } + } finally { + $fingerprintHasher.Dispose() + } +} diff --git a/src/functions/private/Copy-YamlMergeNode.ps1 b/src/functions/private/Copy-YamlMergeNode.ps1 new file mode 100644 index 0000000..6d6f904 --- /dev/null +++ b/src/functions/private/Copy-YamlMergeNode.ps1 @@ -0,0 +1,167 @@ +function Copy-YamlMergeNode { + <# + .SYNOPSIS + Deep-clones a YAML representation graph with identity memoization. + #> + [Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSUseShouldProcessForStateChangingFunctions', '', + Justification = 'Constructs an isolated in-memory representation graph.' + )] + [CmdletBinding()] + [OutputType([pscustomobject])] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Node, + + [Parameter(Mandatory)] + [AllowEmptyCollection()] + [System.Collections.Generic.Dictionary[int, object]] $Cache, + + [Parameter(Mandatory)] + [pscustomobject] $State + ) + + if ($Cache.ContainsKey($Node.Id)) { + Write-Output -InputObject $Cache[$Node.Id] -NoEnumerate + return + } + + $result = [pscustomobject]@{ Value = $null } + $stack = [System.Collections.Generic.Stack[object]]::new() + $stack.Push([pscustomobject]@{ + Source = $Node + Holder = $result + Target = $null + Child = $null + Key = $null + Index = 0 + State = 'Start' + }) + + while ($stack.Count -gt 0) { + $frame = $stack.Peek() + if ($frame.State -eq 'Start') { + if ($Cache.ContainsKey($frame.Source.Id)) { + $frame.Holder.Value = $Cache[$frame.Source.Id] + [void] $stack.Pop() + continue + } + + $State.CreatedNodes++ + if ($State.CreatedNodes -gt $State.MaxNodes) { + throw (New-YamlMergeException -Node $frame.Source ` + -ErrorId 'YamlMergeNodeLimitExceeded' -Message ( + "Cloning the merged YAML graph exceeded the configured limit of $($State.MaxNodes) nodes." + )) + } + $target = New-YamlNode -Id $State.NextId -Kind $frame.Source.Kind ` + -Start $frame.Source.Start -End $frame.Source.End + $State.NextId++ + $target.Tag = [string] $frame.Source.Tag + $target.HasUnknownTag = [bool] $frame.Source.HasUnknownTag + $target.Anchor = [string] $frame.Source.Anchor + $target.Value = $frame.Source.Value + $target.Style = [string] $frame.Source.Style + $target.IsPlainImplicit = [bool] $frame.Source.IsPlainImplicit + $target.IsQuotedImplicit = [bool] $frame.Source.IsQuotedImplicit + $target.MaxNumericLength = [int] $frame.Source.MaxNumericLength + $Cache[$frame.Source.Id] = $target + $frame.Target = $target + $frame.Holder.Value = $target + + if ($frame.Source.Kind -eq 'Scalar') { + [void] $stack.Pop() + } elseif ($frame.Source.Kind -eq 'Alias') { + $frame.Child = [pscustomobject]@{ Value = $null } + $frame.State = 'Alias' + $stack.Push([pscustomobject]@{ + Source = $frame.Source.Target + Holder = $frame.Child + Target = $null + Child = $null + Key = $null + Index = 0 + State = 'Start' + }) + } elseif ($frame.Source.Kind -eq 'Sequence') { + $frame.State = 'Sequence' + } else { + $frame.State = 'MappingKey' + } + continue + } + + if ($frame.State -eq 'Alias') { + $frame.Target.Target = $frame.Child.Value + [void] $stack.Pop() + continue + } + if ($frame.State -eq 'Sequence') { + if ($frame.Index -ge $frame.Source.Items.Count) { + [void] $stack.Pop() + continue + } + $frame.Child = [pscustomobject]@{ Value = $null } + $frame.State = 'SequenceValue' + $stack.Push([pscustomobject]@{ + Source = $frame.Source.Items[$frame.Index] + Holder = $frame.Child + Target = $null + Child = $null + Key = $null + Index = 0 + State = 'Start' + }) + continue + } + if ($frame.State -eq 'SequenceValue') { + $frame.Target.Items.Add($frame.Child.Value) + $frame.Index++ + $frame.State = 'Sequence' + continue + } + if ($frame.State -eq 'MappingKey') { + if ($frame.Index -ge $frame.Source.Entries.Count) { + [void] $stack.Pop() + continue + } + $frame.Child = [pscustomobject]@{ Value = $null } + $frame.State = 'MappingKeyValue' + $stack.Push([pscustomobject]@{ + Source = $frame.Source.Entries[$frame.Index].Key + Holder = $frame.Child + Target = $null + Child = $null + Key = $null + Index = 0 + State = 'Start' + }) + continue + } + if ($frame.State -eq 'MappingKeyValue') { + $frame.Key = $frame.Child.Value + $frame.Child = [pscustomobject]@{ Value = $null } + $frame.State = 'MappingValue' + $stack.Push([pscustomobject]@{ + Source = $frame.Source.Entries[$frame.Index].Value + Holder = $frame.Child + Target = $null + Child = $null + Key = $null + Index = 0 + State = 'Start' + }) + continue + } + if ($frame.State -eq 'MappingValue') { + $frame.Target.Entries.Add([pscustomobject]@{ + Key = $frame.Key + Value = $frame.Child.Value + }) + $frame.Index++ + $frame.State = 'MappingKey' + } + } + + Write-Output -InputObject $result.Value -NoEnumerate +} diff --git a/src/functions/private/Get-YamlMergeFingerprint.ps1 b/src/functions/private/Get-YamlMergeFingerprint.ps1 new file mode 100644 index 0000000..e726e79 --- /dev/null +++ b/src/functions/private/Get-YamlMergeFingerprint.ps1 @@ -0,0 +1,30 @@ +function Get-YamlMergeFingerprint { + <# + .SYNOPSIS + Creates a deterministic candidate index for structural merge comparisons. + #> + [CmdletBinding()] + [OutputType([string])] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Node, + + [Parameter(Mandatory)] + [pscustomobject] $State + ) + + $effective = Get-YamlMergeNode -Node $Node + $tag = Get-YamlMergeNodeTag -Node $effective + if ($effective.Kind -eq 'Scalar') { + $resolved = (Resolve-YamlScalar -Node $effective).Value + $value = Get-YamlScalarFingerprint -Value $resolved -Hasher $State.FingerprintHasher + return 'scalar:{0}:{1}:{2}' -f $tag.Length, $tag, $value + } + + $count = if ($effective.Kind -eq 'Sequence') { + $effective.Items.Count + } else { + $effective.Entries.Count + } + return '{0}:{1}:{2}:{3}' -f $effective.Kind.ToLowerInvariant(), $tag.Length, $tag, $count +} diff --git a/src/functions/private/Get-YamlMergeNode.ps1 b/src/functions/private/Get-YamlMergeNode.ps1 new file mode 100644 index 0000000..ff7212a --- /dev/null +++ b/src/functions/private/Get-YamlMergeNode.ps1 @@ -0,0 +1,18 @@ +function Get-YamlMergeNode { + <# + .SYNOPSIS + Resolves aliases to their effective YAML representation node. + #> + [CmdletBinding()] + [OutputType([pscustomobject])] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Node + ) + + $effective = $Node + while ($effective.Kind -eq 'Alias') { + $effective = $effective.Target + } + Write-Output -InputObject $effective -NoEnumerate +} diff --git a/src/functions/private/Get-YamlMergeNodeTag.ps1 b/src/functions/private/Get-YamlMergeNodeTag.ps1 new file mode 100644 index 0000000..b31c184 --- /dev/null +++ b/src/functions/private/Get-YamlMergeNodeTag.ps1 @@ -0,0 +1,20 @@ +function Get-YamlMergeNodeTag { + <# + .SYNOPSIS + Gets the effective tag used for YAML merge compatibility. + #> + [CmdletBinding()] + [OutputType([string])] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Node + ) + + $effective = Get-YamlMergeNode -Node $Node + $value = if ($effective.Kind -eq 'Scalar') { + (Resolve-YamlScalar -Node $effective).Value + } else { + $null + } + Get-YamlEffectiveTag -Node $effective -Value $value +} diff --git a/src/functions/private/Get-YamlMergePath.ps1 b/src/functions/private/Get-YamlMergePath.ps1 new file mode 100644 index 0000000..20f7be6 --- /dev/null +++ b/src/functions/private/Get-YamlMergePath.ps1 @@ -0,0 +1,29 @@ +function Get-YamlMergePath { + <# + .SYNOPSIS + Creates a stable diagnostic path for one YAML mapping entry. + #> + [CmdletBinding()] + [OutputType([string])] + param ( + [Parameter(Mandatory)] + [string] $Parent, + + [Parameter(Mandatory)] + [pscustomobject] $Key, + + [Parameter(Mandatory)] + [int] $Index + ) + + $effective = Get-YamlMergeNode -Node $Key + if ($effective.Kind -eq 'Scalar' -and + (Get-YamlMergeNodeTag -Node $effective) -ceq 'tag:yaml.org,2002:str') { + $value = [string] (Resolve-YamlScalar -Node $effective).Value + if ($value -cmatch '^[A-Za-z_][A-Za-z0-9_-]*$') { + return "$Parent.$value" + } + return "{0}['{1}']" -f $Parent, $value.Replace("'", "''") + } + return "$Parent{key:$Index}" +} diff --git a/src/functions/private/Merge-YamlRepresentationNode.ps1 b/src/functions/private/Merge-YamlRepresentationNode.ps1 new file mode 100644 index 0000000..39779bd --- /dev/null +++ b/src/functions/private/Merge-YamlRepresentationNode.ps1 @@ -0,0 +1,201 @@ +function Merge-YamlRepresentationNode { + <# + .SYNOPSIS + Merges one later YAML representation node into an existing cloned node. + #> + [Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSUseShouldProcessForStateChangingFunctions', '', + Justification = 'Mutates only an isolated in-memory merge graph.' + )] + [CmdletBinding()] + [OutputType([pscustomobject])] + param ( + [Parameter(Mandatory)] + [pscustomobject] $BaseNode, + + [Parameter(Mandatory)] + [pscustomobject] $OverlayNode, + + [Parameter(Mandatory)] + [ValidateSet('Replace', 'Append', 'Unique')] + [string] $SequenceAction, + + [Parameter(Mandatory)] + [ValidateSet('Replace', 'Error')] + [string] $ConflictAction, + + [Parameter(Mandatory)] + [ValidateSet('Replace', 'Ignore')] + [string] $NullAction, + + [Parameter(Mandatory)] + [string] $Path, + + [Parameter(Mandatory)] + [pscustomobject] $Context + ) + + $Context.WorkState.MergeCount++ + if ($Context.WorkState.MergeCount -gt $Context.WorkState.MaxNodes) { + throw (New-YamlMergeException -Node $OverlayNode -ErrorId 'YamlMergeNodeLimitExceeded' -Message ( + "Merging input index $($Context.InputIndex) document index $($Context.DocumentIndex) " + + "exceeded the configured work limit of $($Context.WorkState.MaxNodes) nodes." + )) + } + + $base = Get-YamlMergeNode -Node $BaseNode + $overlay = Get-YamlMergeNode -Node $OverlayNode + $overlayTag = Get-YamlMergeNodeTag -Node $overlay + if ($NullAction -eq 'Ignore' -and + $overlayTag -ceq 'tag:yaml.org,2002:null') { + Write-Output -InputObject $BaseNode -NoEnumerate + return + } + + if (Test-YamlMergeNodeEqual -Node $base -OtherNode $overlay ` + -State $Context.EqualityState) { + if ($overlay.Kind -ne 'Scalar' -and + -not $Context.CloneCache.ContainsKey($overlay.Id)) { + $Context.CloneCache[$overlay.Id] = $base + } + Write-Output -InputObject $BaseNode -NoEnumerate + return + } + + $baseTag = Get-YamlMergeNodeTag -Node $base + $isCompatible = $base.Kind -ceq $overlay.Kind -and $baseTag -ceq $overlayTag + if (-not $isCompatible) { + if ($ConflictAction -eq 'Error') { + throw (New-YamlMergeException -Node $overlay -ErrorId 'YamlMergeConflict' -Message ( + "YAML merge conflict at $Path in input index $($Context.InputIndex), " + + "document index $($Context.DocumentIndex): $($base.Kind.ToLowerInvariant()) " + + "tag '$baseTag' conflicts with $($overlay.Kind.ToLowerInvariant()) tag '$overlayTag'." + )) + } + return Copy-YamlMergeNode -Node $OverlayNode -Cache $Context.CloneCache ` + -State $Context.CloneState + } + + if ($base.Kind -eq 'Scalar') { + if ($ConflictAction -eq 'Error') { + throw (New-YamlMergeException -Node $overlay -ErrorId 'YamlMergeConflict' -Message ( + "YAML merge conflict at $Path in input index $($Context.InputIndex), " + + "document index $($Context.DocumentIndex): unequal scalar values use tag '$baseTag'." + )) + } + return Copy-YamlMergeNode -Node $OverlayNode -Cache $Context.CloneCache ` + -State $Context.CloneState + } + + if ($base.Kind -eq 'Sequence' -and $SequenceAction -eq 'Replace') { + return Copy-YamlMergeNode -Node $OverlayNode -Cache $Context.CloneCache ` + -State $Context.CloneState + } + + if ($Context.CloneCache.ContainsKey($overlay.Id)) { + if ($OverlayNode.Kind -eq 'Alias') { + return Copy-YamlMergeNode -Node $OverlayNode -Cache $Context.CloneCache ` + -State $Context.CloneState + } + Write-Output -InputObject $Context.CloneCache[$overlay.Id] -NoEnumerate + return + } + $Context.CloneCache[$overlay.Id] = $base + + if ($base.Kind -eq 'Sequence') { + if ($SequenceAction -eq 'Append') { + foreach ($item in $overlay.Items) { + $copy = Copy-YamlMergeNode -Node $item -Cache $Context.CloneCache ` + -State $Context.CloneState + $base.Items.Add($copy) + } + Write-Output -InputObject $BaseNode -NoEnumerate + return + } + + $retained = [System.Collections.Generic.Dictionary[string, object]]::new( + [System.StringComparer]::Ordinal + ) + foreach ($item in $base.Items) { + $fingerprint = Get-YamlMergeFingerprint -Node $item -State $Context.EqualityState + if (-not $retained.ContainsKey($fingerprint)) { + $retained[$fingerprint] = [System.Collections.Generic.List[object]]::new() + } + $retained[$fingerprint].Add($item) + } + foreach ($item in $overlay.Items) { + $fingerprint = Get-YamlMergeFingerprint -Node $item -State $Context.EqualityState + $exists = $false + if ($retained.ContainsKey($fingerprint)) { + foreach ($candidate in $retained[$fingerprint]) { + if (Test-YamlMergeNodeEqual -Node $candidate -OtherNode $item ` + -State $Context.EqualityState) { + $exists = $true + break + } + } + } + if ($exists) { + continue + } + $copy = Copy-YamlMergeNode -Node $item -Cache $Context.CloneCache ` + -State $Context.CloneState + $base.Items.Add($copy) + if (-not $retained.ContainsKey($fingerprint)) { + $retained[$fingerprint] = [System.Collections.Generic.List[object]]::new() + } + $retained[$fingerprint].Add($copy) + } + Write-Output -InputObject $BaseNode -NoEnumerate + return + } + + $entries = [System.Collections.Generic.Dictionary[string, object]]::new( + [System.StringComparer]::Ordinal + ) + foreach ($entry in $base.Entries) { + $fingerprint = Get-YamlMergeFingerprint -Node $entry.Key -State $Context.EqualityState + if (-not $entries.ContainsKey($fingerprint)) { + $entries[$fingerprint] = [System.Collections.Generic.List[object]]::new() + } + $entries[$fingerprint].Add($entry) + } + + for ($index = 0; $index -lt $overlay.Entries.Count; $index++) { + $overlayEntry = $overlay.Entries[$index] + $fingerprint = Get-YamlMergeFingerprint -Node $overlayEntry.Key ` + -State $Context.EqualityState + $match = $null + if ($entries.ContainsKey($fingerprint)) { + foreach ($candidate in $entries[$fingerprint]) { + if (Test-YamlMergeNodeEqual -Node $candidate.Key -OtherNode $overlayEntry.Key ` + -State $Context.EqualityState) { + $match = $candidate + break + } + } + } + + if ($null -ne $match) { + $childPath = Get-YamlMergePath -Parent $Path -Key $overlayEntry.Key -Index $index + $match.Value = Merge-YamlRepresentationNode -BaseNode $match.Value ` + -OverlayNode $overlayEntry.Value -SequenceAction $SequenceAction ` + -ConflictAction $ConflictAction -NullAction $NullAction -Path $childPath ` + -Context $Context + continue + } + + $keyCopy = Copy-YamlMergeNode -Node $overlayEntry.Key -Cache $Context.CloneCache ` + -State $Context.CloneState + $valueCopy = Copy-YamlMergeNode -Node $overlayEntry.Value -Cache $Context.CloneCache ` + -State $Context.CloneState + $newEntry = [pscustomobject]@{ Key = $keyCopy; Value = $valueCopy } + $base.Entries.Add($newEntry) + if (-not $entries.ContainsKey($fingerprint)) { + $entries[$fingerprint] = [System.Collections.Generic.List[object]]::new() + } + $entries[$fingerprint].Add($newEntry) + } + + Write-Output -InputObject $BaseNode -NoEnumerate +} diff --git a/src/functions/private/New-YamlMergeException.ps1 b/src/functions/private/New-YamlMergeException.ps1 new file mode 100644 index 0000000..80e43fe --- /dev/null +++ b/src/functions/private/New-YamlMergeException.ps1 @@ -0,0 +1,30 @@ +function New-YamlMergeException { + <# + .SYNOPSIS + Creates a classified exception for a YAML merge failure. + #> + [Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSUseShouldProcessForStateChangingFunctions', '', + Justification = 'Constructs an in-memory exception without changing system state.' + )] + [CmdletBinding()] + [OutputType([System.FormatException])] + param ( + [Parameter(Mandatory)] + [string] $ErrorId, + + [Parameter(Mandatory)] + [string] $Message, + + [Parameter()] + [AllowNull()] + [pscustomobject] $Node + ) + + if ($null -eq $Node) { + $mark = New-YamlMark -Index 0 -Line 0 -Column 0 + return New-YamlException -Start $mark -End $mark -ErrorId $ErrorId -Message $Message + } + + return New-YamlException -Start $Node.Start -End $Node.End -ErrorId $ErrorId -Message $Message +} diff --git a/src/functions/private/Test-YamlMergeNodeEqual.ps1 b/src/functions/private/Test-YamlMergeNodeEqual.ps1 new file mode 100644 index 0000000..2c8a3d2 --- /dev/null +++ b/src/functions/private/Test-YamlMergeNodeEqual.ps1 @@ -0,0 +1,180 @@ +function Test-YamlMergeNodeEqual { + <# + .SYNOPSIS + Compares YAML representation graphs with collision-safe structural equality. + #> + [CmdletBinding()] + [OutputType([bool])] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Node, + + [Parameter(Mandatory)] + [pscustomobject] $OtherNode, + + [Parameter(Mandatory)] + [pscustomobject] $State + ) + + $seen = [System.Collections.Generic.HashSet[string]]::new( + [System.StringComparer]::Ordinal + ) + $pending = [System.Collections.Generic.Stack[object]]::new() + $pending.Push([pscustomobject]@{ Left = $Node; Right = $OtherNode }) + + while ($pending.Count -gt 0) { + $pair = $pending.Pop() + $left = Get-YamlMergeNode -Node $pair.Left + $right = Get-YamlMergeNode -Node $pair.Right + $pairId = '{0}:{1}' -f $left.Id, $right.Id + if (-not $seen.Add($pairId)) { + continue + } + + $State.EqualityCount++ + if ($State.EqualityCount -gt $State.MaxNodes) { + throw (New-YamlMergeException -Node $right -ErrorId 'YamlMergeEqualityLimitExceeded' -Message ( + "YAML structural equality exceeded the configured limit of $($State.MaxNodes) node pairs." + )) + } + + if ($left.Kind -cne $right.Kind) { + return $false + } + $leftTag = Get-YamlMergeNodeTag -Node $left + $rightTag = Get-YamlMergeNodeTag -Node $right + if ($leftTag -cne $rightTag) { + return $false + } + + if ($left.Kind -eq 'Scalar') { + $leftValue = (Resolve-YamlScalar -Node $left).Value + $rightValue = (Resolve-YamlScalar -Node $right).Value + if ($null -eq $leftValue -or $null -eq $rightValue) { + if ($null -ne $leftValue -or $null -ne $rightValue) { + return $false + } + continue + } + if ($leftValue -is [byte[]] -or $rightValue -is [byte[]]) { + if ($leftValue -isnot [byte[]] -or $rightValue -isnot [byte[]] -or + $leftValue.Count -ne $rightValue.Count) { + return $false + } + for ($index = 0; $index -lt $leftValue.Count; $index++) { + if ($leftValue[$index] -ne $rightValue[$index]) { + return $false + } + } + continue + } + if ($leftValue -is [datetimeoffset] -or $leftValue -is [datetime] -or + $rightValue -is [datetimeoffset] -or $rightValue -is [datetime]) { + $leftTicks = if ($leftValue -is [datetimeoffset]) { + $leftValue.UtcDateTime.Ticks + } elseif ($leftValue.Kind -eq [System.DateTimeKind]::Local) { + $leftValue.ToUniversalTime().Ticks + } else { + $leftValue.Ticks + } + $rightTicks = if ($rightValue -is [datetimeoffset]) { + $rightValue.UtcDateTime.Ticks + } elseif ($rightValue.Kind -eq [System.DateTimeKind]::Local) { + $rightValue.ToUniversalTime().Ticks + } else { + $rightValue.Ticks + } + if ($leftTicks -ne $rightTicks) { + return $false + } + continue + } + if ($leftValue -is [decimal] -or $leftValue -is [double] -or + $leftValue -is [single] -or $rightValue -is [decimal] -or + $rightValue -is [double] -or $rightValue -is [single]) { + $leftNumber = Get-YamlNormalizedFloat -Value $leftValue + $rightNumber = Get-YamlNormalizedFloat -Value $rightValue + if ($leftNumber -cne $rightNumber) { + return $false + } + continue + } + if ($leftValue -is [string] -and $rightValue -is [string]) { + if (-not $leftValue.Equals($rightValue, [System.StringComparison]::Ordinal)) { + return $false + } + continue + } + if ($leftValue -is [bool] -and $rightValue -is [bool]) { + if ($leftValue -ne $rightValue) { + return $false + } + continue + } + $leftText = $leftValue.ToString([System.Globalization.CultureInfo]::InvariantCulture) + $rightText = $rightValue.ToString([System.Globalization.CultureInfo]::InvariantCulture) + if ($leftText -cne $rightText) { + return $false + } + continue + } + + if ($left.Kind -eq 'Sequence') { + if ($left.Items.Count -ne $right.Items.Count) { + return $false + } + for ($index = $left.Items.Count - 1; $index -ge 0; $index--) { + $pending.Push([pscustomobject]@{ + Left = $left.Items[$index] + Right = $right.Items[$index] + }) + } + continue + } + + if ($left.Entries.Count -ne $right.Entries.Count) { + return $false + } + $rightEntries = [System.Collections.Generic.Dictionary[string, object]]::new( + [System.StringComparer]::Ordinal + ) + for ($index = 0; $index -lt $right.Entries.Count; $index++) { + $fingerprint = Get-YamlMergeFingerprint -Node $right.Entries[$index].Key -State $State + if (-not $rightEntries.ContainsKey($fingerprint)) { + $rightEntries[$fingerprint] = [System.Collections.Generic.List[object]]::new() + } + $rightEntries[$fingerprint].Add([pscustomobject]@{ + Index = $index + Entry = $right.Entries[$index] + }) + } + $matched = [System.Collections.Generic.HashSet[int]]::new() + foreach ($leftEntry in $left.Entries) { + $fingerprint = Get-YamlMergeFingerprint -Node $leftEntry.Key -State $State + if (-not $rightEntries.ContainsKey($fingerprint)) { + return $false + } + $match = $null + foreach ($candidate in $rightEntries[$fingerprint]) { + if ($matched.Contains($candidate.Index)) { + continue + } + if (Test-YamlMergeNodeEqual -Node $leftEntry.Key -OtherNode $candidate.Entry.Key ` + -State $State) { + $match = $candidate + break + } + } + if ($null -eq $match) { + return $false + } + [void] $matched.Add($match.Index) + $pending.Push([pscustomobject]@{ + Left = $leftEntry.Value + Right = $match.Entry.Value + }) + } + } + + return $true +} diff --git a/src/functions/public/Merge-Yaml.ps1 b/src/functions/public/Merge-Yaml.ps1 new file mode 100644 index 0000000..dabccc4 --- /dev/null +++ b/src/functions/public/Merge-Yaml.ps1 @@ -0,0 +1,258 @@ +function Merge-Yaml { + <# + .SYNOPSIS + Merges two or more complete YAML streams. + + .DESCRIPTION + Parses every input string as one complete YAML stream, merges documents + pairwise by zero-based document index, and emits one deterministic YAML + string directly from the resulting representation graphs. Every stream + must contain the same positive number of documents. Later streams have + higher precedence. + + Compatible mappings merge recursively by YAML structural key equality. + Base key order is retained, replaced values keep their position, and new + overlay keys append in overlay order. Tags, complex keys, anchors, + aliases, shared nodes, and recursive graphs remain representation data; + values are never projected through PowerShell objects or dictionaries. + + Output uses LF line endings, has no final newline, and explicitly starts + every document. Input array elements and pipeline records are complete + streams, not individual lines. Use Get-Content -Raw or Import-Yaml file + handling as appropriate. + + .PARAMETER InputObject + Two or more complete YAML stream strings. Each array element or pipeline + record is parsed independently as one stream. + + .PARAMETER SequenceAction + Action for unequal sequences with compatible effective tags. Replace uses + the later sequence, Append concatenates entries, and Unique appends only + structurally new entries. The default is Replace. + + .PARAMETER ConflictAction + Action for unequal scalars, collection-kind differences, or incompatible + effective tags. Replace uses the later node and Error terminates with + document, path, and input context. The default is Replace. + + .PARAMETER NullAction + Action when a later existing node has the YAML null effective tag. + Replace applies normal merge and conflict behavior; Ignore retains the + prior node. The default is Replace. + + .PARAMETER Indent + Block indentation from 2 through 9 spaces. The default is 2. + + .PARAMETER Depth + Maximum YAML node nesting depth. The default is 100. + + .PARAMETER MaxNodes + Maximum nodes per parsed stream and the invocation-wide ceiling for + clone, equality, merge, and result work. The default is 100000. + + .PARAMETER MaxAliases + Maximum aliases per parsed stream and in the result. The default is 1000. + + .PARAMETER MaxScalarLength + Maximum decoded character count for one scalar. The default is 1048576. + + .PARAMETER MaxTagLength + Maximum expanded character count for one tag. The default is 1024. + + .PARAMETER MaxTotalTagLength + Maximum cumulative expanded tag characters per input and in the result. + The default is 65536. + + .PARAMETER MaxNumericLength + Maximum digits in an implicitly or explicitly typed number. The default + is 4096. + + .EXAMPLE + $merged = Merge-Yaml -InputObject @($baseYaml, $overlayYaml) + + Merges two complete stream strings with the overlay taking precedence. + + .EXAMPLE + $baseYaml, $environmentYaml, $secretYaml | Merge-Yaml -SequenceAction Unique + + Merges three complete pipeline stream records and deduplicates compatible + sequence entries structurally. + + .EXAMPLE + $base = Get-Content -LiteralPath '.\base.yaml' -Raw + $overlay = Get-Content -LiteralPath '.\overlay.yaml' -Raw + Merge-Yaml -InputObject @($base, $overlay) -ConflictAction Error -Indent 4 + + Reads complete files and rejects incompatible merge conflicts. + + .INPUTS + System.String[] + + .OUTPUTS + System.String + + .NOTES + YAML 1.1 merge keys are ordinary mapping data and are never expanded. + + .LINK + Format-Yaml + #> + [OutputType([string])] + [CmdletBinding()] + param ( + [Parameter(Mandatory, Position = 0, ValueFromPipeline)] + [AllowEmptyString()] + [string[]] $InputObject, + + [Parameter()] + [ValidateSet('Replace', 'Append', 'Unique')] + [string] $SequenceAction = 'Replace', + + [Parameter()] + [ValidateSet('Replace', 'Error')] + [string] $ConflictAction = 'Replace', + + [Parameter()] + [ValidateSet('Replace', 'Ignore')] + [string] $NullAction = 'Replace', + + [Parameter()] + [ValidateRange(2, 9)] + [int] $Indent = 2, + + [Parameter()] + [ValidateRange(1, 128)] + [int] $Depth = 100, + + [Parameter()] + [ValidateRange(1, 2147483647)] + [int] $MaxNodes = 100000, + + [Parameter()] + [ValidateRange(0, 2147483647)] + [int] $MaxAliases = 1000, + + [Parameter()] + [ValidateRange(1, 2147483647)] + [int] $MaxScalarLength = 1048576, + + [Parameter()] + [ValidateRange(1, 1048576)] + [int] $MaxTagLength = 1024, + + [Parameter()] + [ValidateRange(1, 2147483647)] + [int] $MaxTotalTagLength = 65536, + + [Parameter()] + [ValidateRange(1, 1048576)] + [int] $MaxNumericLength = 4096 + ) + + begin { + $streams = [System.Collections.Generic.List[string]]::new() + } + process { + foreach ($stream in $InputObject) { + $streams.Add($stream) + } + } + end { + $fingerprintHasher = $null + try { + if ($streams.Count -lt 2) { + throw (New-YamlMergeException -ErrorId 'YamlMergeInputCount' -Message ( + "Merge-Yaml requires at least two complete YAML streams; received $($streams.Count)." + )) + } + + $parsedStreams = [System.Collections.Generic.List[object]]::new() + $expectedDocumentCount = -1 + for ($inputIndex = 0; $inputIndex -lt $streams.Count; $inputIndex++) { + $documentBox = Read-YamlStream -Yaml $streams[$inputIndex] -Depth $Depth ` + -MaxNodes $MaxNodes -MaxAliases $MaxAliases ` + -MaxScalarLength $MaxScalarLength -MaxTagLength $MaxTagLength ` + -MaxTotalTagLength $MaxTotalTagLength -MaxNumericLength $MaxNumericLength + $documents = [object[]] $documentBox.Value + if ($documents.Count -eq 0) { + throw (New-YamlMergeException -ErrorId 'YamlMergeEmptyStream' -Message ( + "YAML input index $inputIndex contains 0 documents; every stream must contain " + + 'the same positive document count.' + )) + } + if ($inputIndex -eq 0) { + $expectedDocumentCount = $documents.Count + } elseif ($documents.Count -ne $expectedDocumentCount) { + throw (New-YamlMergeException ` + -ErrorId 'YamlMergeDocumentCountMismatch' -Message ( + "YAML input index $inputIndex contains $($documents.Count) documents; " + + "expected $expectedDocumentCount." + )) + } + $parsedStreams.Add($documents) + } + + $cloneState = [pscustomobject]@{ + NextId = 1 + CreatedNodes = 0 + MaxNodes = $MaxNodes + } + $resultDocuments = [System.Collections.Generic.List[object]]::new() + $baseCache = [System.Collections.Generic.Dictionary[int, object]]::new() + foreach ($document in $parsedStreams[0]) { + $resultDocuments.Add(( + Copy-YamlMergeNode -Node $document -Cache $baseCache -State $cloneState + )) + } + + $fingerprintHasher = [System.Security.Cryptography.SHA256]::Create() + $equalityState = [pscustomobject]@{ + EqualityCount = 0 + MaxNodes = $MaxNodes + FingerprintHasher = $fingerprintHasher + } + $workState = [pscustomobject]@{ + MergeCount = 0 + MaxNodes = $MaxNodes + } + for ($inputIndex = 1; $inputIndex -lt $parsedStreams.Count; $inputIndex++) { + $cloneCache = [System.Collections.Generic.Dictionary[int, object]]::new() + foreach ($documentIndex in 0..($expectedDocumentCount - 1)) { + $context = [pscustomobject]@{ + InputIndex = $inputIndex + DocumentIndex = $documentIndex + CloneCache = $cloneCache + CloneState = $cloneState + EqualityState = $equalityState + WorkState = $workState + } + $resultDocuments[$documentIndex] = Merge-YamlRepresentationNode ` + -BaseNode $resultDocuments[$documentIndex] ` + -OverlayNode $parsedStreams[$inputIndex][$documentIndex] ` + -SequenceAction $SequenceAction -ConflictAction $ConflictAction ` + -NullAction $NullAction -Path '$' -Context $context + } + } + + $resultArray = [object[]] $resultDocuments.ToArray() + Assert-YamlMergeGraph -Documents $resultArray -Depth $Depth -MaxNodes $MaxNodes ` + -MaxAliases $MaxAliases -MaxScalarLength $MaxScalarLength ` + -MaxTagLength $MaxTagLength -MaxTotalTagLength $MaxTotalTagLength + $merged = ConvertTo-YamlRepresentationText -Documents $resultArray -Indent $Indent + $PSCmdlet.WriteObject($merged, $false) + } catch { + $failure = $_ + if (-not $failure.Exception.Data.Contains('IsYamlException')) { + throw + } + $record = New-YamlErrorRecord -Exception $failure.Exception ` + -DefaultErrorId 'YamlMergeFailed' -Category InvalidData ` + -TargetObject $streams.ToArray() + $PSCmdlet.ThrowTerminatingError($record) + } finally { + if ($null -ne $fingerprintHasher) { + $fingerprintHasher.Dispose() + } + } + } +} diff --git a/tests/Merge-Yaml.Tests.ps1 b/tests/Merge-Yaml.Tests.ps1 new file mode 100644 index 0000000..1b677e3 --- /dev/null +++ b/tests/Merge-Yaml.Tests.ps1 @@ -0,0 +1,594 @@ +#Requires -Modules @{ ModuleName = 'Pester'; ModuleVersion = '6.0.0'; MaximumVersion = '6.*' } + +[Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSReviewUnusedParameter', '', + Justification = 'Required for Pester tests' +)] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSUseDeclaredVarsMoreThanAssignments', '', + Justification = 'Required for Pester tests' +)] +[CmdletBinding()] +param() + +BeforeAll { + . (Join-Path $PSScriptRoot 'TestBootstrap.ps1') + + function Get-MergeYamlFailure { + <# + .SYNOPSIS + Captures one expected terminating Merge-Yaml error. + #> + param ( + [Parameter(Mandatory)] + [scriptblock] $Action + ) + + try { + $null = & $Action + } catch { + return $_ + } + throw 'The YAML merge unexpectedly succeeded.' + } + + function Get-MergeYamlGraphFact { + <# + .SYNOPSIS + Returns representation graph facts without projecting YAML values. + #> + param ( + [Parameter(Mandatory)] + [string] $Yaml + ) + + $implementation = { + param ([string] $YamlText) + + $documents = (Read-YamlStreamCore -Yaml $YamlText -Depth 100 -MaxNodes 10000 ` + -MaxAliases 1000 -MaxScalarLength 1048576 -MaxTagLength 1024 ` + -MaxTotalTagLength 65536 -MaxNumericLength 4096).Value + $visited = [System.Collections.Generic.HashSet[int]]::new() + $stack = [System.Collections.Generic.Stack[object]]::new() + foreach ($document in $documents) { + $stack.Push($document) + } + $aliases = 0 + $nodes = 0 + $tags = [System.Collections.Generic.List[string]]::new() + while ($stack.Count -gt 0) { + $node = $stack.Pop() + if (-not $visited.Add($node.Id)) { + continue + } + $nodes++ + if (-not [string]::IsNullOrEmpty($node.Tag)) { + $tags.Add([string] $node.Tag) + } + if ($node.Kind -eq 'Alias') { + $aliases++ + $stack.Push($node.Target) + } elseif ($node.Kind -eq 'Sequence') { + foreach ($item in $node.Items) { + $stack.Push($item) + } + } elseif ($node.Kind -eq 'Mapping') { + foreach ($entry in $node.Entries) { + $stack.Push($entry.Value) + $stack.Push($entry.Key) + } + } + } + + [pscustomobject]@{ + DocumentCount = $documents.Count + NodeCount = $nodes + AliasCount = $aliases + Tags = [string[]] $tags.ToArray() + } + } + + $loadedModule = Get-Module -Name Yaml | Select-Object -First 1 + if ($null -eq $loadedModule) { + return & $implementation $Yaml + } + return & $loadedModule $implementation $Yaml + } +} + +Describe 'Merge-Yaml' { + Context 'Public contract' { + It 'exposes the documented advanced string merge contract' { + $command = Get-Command -Name Merge-Yaml + $inputParameter = $command.Parameters['InputObject'] + $inputAttribute = $inputParameter.Attributes | + Where-Object { $_ -is [System.Management.Automation.ParameterAttribute] } + $allowEmptyString = $inputParameter.Attributes | + Where-Object { $_ -is [System.Management.Automation.AllowEmptyStringAttribute] } + $indentRange = $command.Parameters['Indent'].Attributes | + Where-Object { $_ -is [System.Management.Automation.ValidateRangeAttribute] } + $sequenceSet = $command.Parameters['SequenceAction'].Attributes | + Where-Object { $_ -is [System.Management.Automation.ValidateSetAttribute] } + $conflictSet = $command.Parameters['ConflictAction'].Attributes | + Where-Object { $_ -is [System.Management.Automation.ValidateSetAttribute] } + $nullSet = $command.Parameters['NullAction'].Attributes | + Where-Object { $_ -is [System.Management.Automation.ValidateSetAttribute] } + $help = Get-Help -Name Merge-Yaml -Full + + $command.CmdletBinding | Should -BeTrue + $inputParameter.ParameterType | Should -Be ([string[]]) + $inputAttribute.Mandatory | Should -BeTrue + $inputAttribute.Position | Should -Be 0 + $inputAttribute.ValueFromPipeline | Should -BeTrue + $allowEmptyString | Should -Not -BeNullOrEmpty + @($command.OutputType.Type) | Should -Contain ([string]) + @($sequenceSet.ValidValues) | Should -Be @('Replace', 'Append', 'Unique') + @($conflictSet.ValidValues) | Should -Be @('Replace', 'Error') + @($nullSet.ValidValues) | Should -Be @('Replace', 'Ignore') + $indentRange.MinRange | Should -Be 2 + $indentRange.MaxRange | Should -Be 9 + $help.Synopsis | Should -Not -BeNullOrEmpty + $help.Description.Text | Should -Match 'complete YAML stream' + @($help.Examples.Example).Count | Should -BeGreaterOrEqual 2 + } + + It 'mirrors every parser safety range' -ForEach @( + @{ Name = 'Depth'; Minimum = 1; Maximum = 128 } + @{ Name = 'MaxNodes'; Minimum = 1; Maximum = 2147483647 } + @{ Name = 'MaxAliases'; Minimum = 0; Maximum = 2147483647 } + @{ Name = 'MaxScalarLength'; Minimum = 1; Maximum = 2147483647 } + @{ Name = 'MaxTagLength'; Minimum = 1; Maximum = 1048576 } + @{ Name = 'MaxTotalTagLength'; Minimum = 1; Maximum = 2147483647 } + @{ Name = 'MaxNumericLength'; Minimum = 1; Maximum = 1048576 } + ) { + $range = (Get-Command Merge-Yaml).Parameters[$Name].Attributes | + Where-Object { $_ -is [System.Management.Automation.ValidateRangeAttribute] } + + $range.MinRange | Should -Be $Minimum + $range.MaxRange | Should -Be $Maximum + } + + It 'accepts direct arrays and complete pipeline stream records' { + $base = "root:`n first: 1" + $overlay = "root:`n second: 2" + $expected = "root:`n first: 1`n second: 2" | Format-Yaml + + (Merge-Yaml -InputObject @($base, $overlay)) | Should -BeExactly $expected + (@($base, $overlay) | Merge-Yaml) | Should -BeExactly $expected + } + + It 'emits exactly one LF-only string without a final newline' { + $result = @(Merge-Yaml -InputObject @("one: 1`r`n", "two: 2`r`n")) + + $result.Count | Should -Be 1 + $result[0] | Should -BeOfType [string] + $result[0] | Should -Not -Match "`r" + $result[0].EndsWith("`n", [System.StringComparison]::Ordinal) | + Should -BeFalse + } + + It 'requires at least two complete streams' { + $failure = Get-MergeYamlFailure { Merge-Yaml -InputObject 'one: 1' } + + $failure.Exception.Data['YamlErrorId'] | Should -BeExactly 'YamlMergeInputCount' + $failure.FullyQualifiedErrorId | Should -Be 'YamlMergeInputCount,Merge-Yaml' + } + } + + Context 'Mapping precedence and order' { + It 'merges mappings recursively while retaining and appending key positions' { + $base = @' +root: + keep: 1 + replace: old +tail: base +'@ + $overlay = @' +root: + replace: new + added: true +'@ + $expected = @' +root: + keep: 1 + replace: new + added: true +tail: base +'@ | Format-Yaml + + (Merge-Yaml -InputObject @($base, $overlay)) | Should -BeExactly $expected + } + + It 'applies later precedence across three or more streams' { + $first = "value: first`nfirst: true" + $second = "value: second`nsecond: true" + $third = "value: third`nthird: true" + $expected = "value: third`nfirst: true`nsecond: true`nthird: true" | Format-Yaml + + (Merge-Yaml $first, $second, $third) | Should -BeExactly $expected + } + + It 'merges structurally equal complex mapping keys' { + $base = @' +? [region, { port: 443 }] +: { first: 1 } +'@ + $overlay = @' +? + - region + - port: 443 +: { second: 2 } +'@ + $merged = Merge-Yaml $base, $overlay + $result = $merged | ConvertFrom-Yaml -AsHashtable + $entry = @($result.GetEnumerator())[0] + + $result.Count | Should -Be 1 + $entry.Value['first'] | Should -Be 1 + $entry.Value['second'] | Should -Be 2 + } + + It 'keeps YAML 1.1 merge syntax as ordinary mapping data' { + $base = @' +<<: + fromBase: true +ordinary: value +'@ + $overlay = @' +<<: + fromOverlay: true +'@ + $result = Merge-Yaml $base, $overlay | ConvertFrom-Yaml -AsHashtable + + @($result.Keys) | Should -Be @('<<', 'ordinary') + $result['<<']['fromBase'] | Should -BeTrue + $result['<<']['fromOverlay'] | Should -BeTrue + $result.Contains('fromBase') | Should -BeFalse + $result.Contains('fromOverlay') | Should -BeFalse + } + } + + Context 'Sequence policies' { + It 'applies the requested compatible-sequence action' -ForEach @( + @{ + Action = 'Replace' + Expected = "items:`n- two`n- three" + } + @{ + Action = 'Append' + Expected = "items:`n- one`n- two`n- two`n- three" + } + @{ + Action = 'Unique' + Expected = "items:`n- one`n- two`n- three" + } + ) { + $base = 'items: [one, two]' + $overlay = 'items: [two, three]' + + $actual = Merge-Yaml $base, $overlay -SequenceAction $Action + + $actual | Should -BeExactly ($Expected | Format-Yaml) + } + + It 'retains equal sequences under every action' -ForEach @( + @{ Action = 'Replace' } + @{ Action = 'Append' } + @{ Action = 'Unique' } + ) { + $base = "items: &items [one, two]`ncopy: *items" + $merged = Merge-Yaml $base, $base -SequenceAction $Action -ConflictAction Error + $result = $merged | ConvertFrom-Yaml -AsHashtable + + $merged | Should -BeExactly ($base | Format-Yaml) + [object]::ReferenceEquals($result['items'], $result['copy']) | + Should -BeTrue + } + + It 'deduplicates scalar, complex, tagged, and cyclic values structurally' { + $base = @' +items: + - one + - { key: value } + - !item tagged + - &cycle { self: *cycle } +'@ + $overlay = @' +items: + - one + - key: value + - !item tagged + - &other { self: *other } + - added +'@ + $merged = Merge-Yaml $base, $overlay -SequenceAction Unique + $result = $merged | ConvertFrom-Yaml -AsHashtable + $facts = Get-MergeYamlGraphFact -Yaml $merged + + $result['items'].Count | Should -Be 5 + $result['items'][4] | Should -Be 'added' + [object]::ReferenceEquals( + $result['items'][3], + $result['items'][3]['self'] + ) | Should -BeTrue + @($facts.Tags) | Should -Contain '!item' + } + + It 'preserves overlay sharing and cycles while appending' { + $base = 'items: [base]' + $overlay = @' +items: + - &shared { value: overlay } + - *shared + - &cycle { self: *cycle } +'@ + $merged = Merge-Yaml $base, $overlay -SequenceAction Append + $result = $merged | ConvertFrom-Yaml -AsHashtable + + [object]::ReferenceEquals($result['items'][1], $result['items'][2]) | + Should -BeTrue + [object]::ReferenceEquals( + $result['items'][3], + $result['items'][3]['self'] + ) | Should -BeTrue + } + } + + Context 'Conflict and null policies' { + It 'replaces incompatible scalar values by default' { + (Merge-Yaml 'value: old', 'value: new') | + Should -BeExactly ('value: new' | Format-Yaml) + } + + It 'rejects unequal scalar, kind, and effective-tag conflicts with context' -ForEach @( + @{ Base = 'value: old'; Overlay = 'value: new'; Path = '\$\.value' } + @{ Base = 'value: { key: one }'; Overlay = 'value: [one]'; Path = '\$\.value' } + @{ Base = '!first { key: one }'; Overlay = '!second { key: one }'; Path = '\$' } + ) { + $failure = Get-MergeYamlFailure { + Merge-Yaml $Base, $Overlay -ConflictAction Error + } + + $failure.Exception.Data['YamlErrorId'] | Should -BeExactly 'YamlMergeConflict' + $failure.Exception.Message | Should -Match 'input index 1' + $failure.Exception.Message | Should -Match 'document index 0' + $failure.Exception.Message | Should -Match $Path + } + + It 'retains structurally equal nodes under Error conflict policy' { + $base = '!same { nested: [one, { two: 2 }] }' + $overlay = '!same { nested: [one, { two: 2 }] }' + + (Merge-Yaml $base, $overlay -ConflictAction Error) | + Should -BeExactly ($base | Format-Yaml) + } + + It 'applies nested null replacement and ignoring' { + $base = "root:`n value: retained" + $overlay = "root:`n value: null" + + $replace = Merge-Yaml $base, $overlay | ConvertFrom-Yaml -AsHashtable + $ignore = Merge-Yaml $base, $overlay -NullAction Ignore | + ConvertFrom-Yaml -AsHashtable + + $replace['root']['value'] | Should -BeNullOrEmpty + $ignore['root']['value'] | Should -Be 'retained' + } + + It 'applies root null replacement and ignoring' { + $base = 'value: retained' + + (Merge-Yaml $base, 'null' | ConvertFrom-Yaml) | Should -BeNullOrEmpty + (Merge-Yaml $base, 'null' -NullAction Ignore) | + Should -BeExactly ($base | Format-Yaml) + } + } + + Context 'Tags and graph identity' { + It 'recursively merges compatible standard and unknown mapping tags' -ForEach @( + @{ + Base = '!!map { first: 1 }' + Overlay = '{ second: 2 }' + ExpectedTag = 'tag:yaml.org,2002:map' + } + @{ + Base = '! { first: 1 }' + Overlay = '! { second: 2 }' + ExpectedTag = 'tag:example.test,2026:item' + } + ) { + $merged = Merge-Yaml $Base, $Overlay + $root = Get-TestYamlRepresentationRoot -Yaml $merged + $projected = $merged | ConvertFrom-Yaml -AsHashtable + + $root.Tag | Should -BeExactly $ExpectedTag + $projected['first'] | Should -Be 1 + $projected['second'] | Should -Be 2 + } + + It 'preserves base aliases when their target is recursively merged' { + $base = @' +root: &shared + base: true +copy: *shared +'@ + $overlay = @' +root: + overlay: true +'@ + $result = Merge-Yaml $base, $overlay | ConvertFrom-Yaml -AsHashtable + + $result['root']['base'] | Should -BeTrue + $result['copy']['overlay'] | Should -BeTrue + [object]::ReferenceEquals($result['root'], $result['copy']) | + Should -BeTrue + } + + It 'preserves shared overlay mappings across matched base entries' { + $base = @' +first: { base: one } +second: { base: two } +'@ + $overlay = @' +first: &shared { overlay: true } +second: *shared +'@ + $result = Merge-Yaml $base, $overlay | ConvertFrom-Yaml -AsHashtable + + $result['first']['overlay'] | Should -BeTrue + [object]::ReferenceEquals($result['first'], $result['second']) | + Should -BeTrue + } + + It 'preserves a recursively merged overlay cycle' { + $base = 'node: { base: true }' + $overlay = @' +node: &cycle + overlay: true + self: *cycle +'@ + $merged = Merge-Yaml $base, $overlay + $result = $merged | ConvertFrom-Yaml -AsHashtable + + $result['node']['base'] | Should -BeTrue + $result['node']['overlay'] | Should -BeTrue + [object]::ReferenceEquals($result['node'], $result['node']['self']) | + Should -BeTrue + $merged | Test-Yaml | Should -BeTrue + } + } + + Context 'Documents, validation, and limits' { + It 'merges equal positive multi-document streams by document index' { + $base = "---`nfirst: base`n---`nsecond: base" + $overlay = "---`nfirstOverlay: true`n---`nsecond: overlay" + $merged = Merge-Yaml $base, $overlay + $documents = @($merged | ConvertFrom-Yaml -AsHashtable) + + $documents.Count | Should -Be 2 + $documents[0]['first'] | Should -Be 'base' + $documents[0]['firstOverlay'] | Should -BeTrue + $documents[1]['second'] | Should -Be 'overlay' + } + + It 'rejects no-document streams with their zero-based input index' -ForEach @( + @{ Empty = '' } + @{ Empty = "# comment only`n" } + ) { + $failure = Get-MergeYamlFailure { + Merge-Yaml 'value: base', $Empty + } + + $failure.Exception.Data['YamlErrorId'] | Should -BeExactly 'YamlMergeEmptyStream' + $failure.Exception.Message | Should -Match 'input index 1' + $failure.Exception.Message | Should -Match '0 documents' + } + + It 'rejects document-count mismatches with expected and actual counts' { + $failure = Get-MergeYamlFailure { + Merge-Yaml 'one: 1', "---`ntwo: 2`n---`nthree: 3" + } + + $failure.Exception.Data['YamlErrorId'] | + Should -BeExactly 'YamlMergeDocumentCountMismatch' + $failure.Exception.Message | Should -Match 'input index 1' + $failure.Exception.Message | Should -Match '2 documents' + $failure.Exception.Message | Should -Match 'expected 1' + } + + It 'preserves parser invalid-input classifications' { + $failure = Get-MergeYamlFailure { + Merge-Yaml 'valid: true', '[invalid' + } + + $failure.Exception.Data['YamlErrorId'] | + Should -BeExactly 'YamlInvalidFlowCollection' + $failure.FullyQualifiedErrorId | + Should -Be 'YamlInvalidFlowCollection,Merge-Yaml' + } + + It 'enforces clone and result node budgets' { + $failure = Get-MergeYamlFailure { + Merge-Yaml 'a: 1', 'b: 2' -MaxNodes 3 + } + + $failure.Exception.Data['YamlErrorId'] | + Should -BeExactly 'YamlMergeNodeLimitExceeded' + } + + It 'enforces cumulative equality work budgets' { + $base = '[{ a: 1 }, { b: 2 }]' + $overlay = '[{ b: 2 }, { a: 1 }]' + $failure = Get-MergeYamlFailure { + Merge-Yaml $base, $overlay -SequenceAction Unique -MaxNodes 7 + } + + $failure.Exception.Data['YamlErrorId'] | + Should -BeExactly 'YamlMergeEqualityLimitExceeded' + } + + It 'enforces resulting alias and tag budgets' -ForEach @( + @{ + Base = "a: &a one`nb: *a" + Overlay = "c: &c two`nd: *c" + Parameters = @{ MaxAliases = 1 } + ErrorId = 'YamlMergeAliasLimitExceeded' + } + @{ + Base = 'a: !x one' + Overlay = 'b: !y two' + Parameters = @{ MaxTotalTagLength = 2 } + ErrorId = 'YamlMergeTagLimitExceeded' + } + ) { + $failure = Get-MergeYamlFailure { + Merge-Yaml $Base, $Overlay @Parameters + } + + $failure.Exception.Data['YamlErrorId'] | Should -BeExactly $ErrorId + } + } + + Context 'Determinism and representation smoke coverage' { + It 'is deterministic, leaves source text semantics unchanged, and self-parses' { + $base = @' +root: &root + first: 1 +copy: *root +'@ + $overlay = @' +root: + second: [two, three] +'@ + $baseBefore = $base | Format-Yaml + $first = Merge-Yaml $base, $overlay -SequenceAction Unique -Indent 4 + $second = Merge-Yaml $base, $overlay -SequenceAction Unique -Indent 4 + + $second | Should -BeExactly $first + ($base | Format-Yaml) | Should -BeExactly $baseBefore + $first | Test-Yaml | Should -BeTrue + $first | Should -BeExactly ($first | Format-Yaml -Indent 4) + } + + It 'retains representation features from selected corpus fixtures' -ForEach @( + @{ Fixture = '2JQS.yaml' } + @{ Fixture = '6BFJ.yaml' } + @{ Fixture = '565N.yaml' } + @{ Fixture = 'SBG9.yaml' } + ) { + $path = Join-Path $PSScriptRoot "fixtures\yaml-test-suite\$Fixture" + $yaml = Get-Content -LiteralPath $path -Raw + + if ($Fixture -eq '2JQS.yaml') { + { Merge-Yaml $yaml, $yaml } | Should -Throw + } else { + $merged = Merge-Yaml $yaml, $yaml -ConflictAction Error + $facts = Get-MergeYamlGraphFact -Yaml $merged + + $merged | Test-Yaml | Should -BeTrue + $facts.DocumentCount | Should -BeGreaterThan 0 + $merged | Should -BeExactly ($merged | Format-Yaml) + } + } + } +} From 7f1d7805411acaaf99949dde9a07452883e3c67f Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sat, 25 Jul 2026 07:15:40 +0200 Subject: [PATCH 2/8] Verify YAML merge package exports Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/Packaging.Tests.ps1 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/Packaging.Tests.ps1 b/tests/Packaging.Tests.ps1 index 7e30f94..6e62607 100644 --- a/tests/Packaging.Tests.ps1 +++ b/tests/Packaging.Tests.ps1 @@ -134,6 +134,7 @@ Describe 'Generated artifact package' { 'Export-Yaml', 'Format-Yaml', 'Import-Yaml', + 'Merge-Yaml', 'Test-Yaml' ) @($manifest.FileList) | Should -Contain 'Yaml.psm1' @@ -176,6 +177,15 @@ $formatted = '{name: Ada, active: true}' | Format-Yaml if ($formatted -cne "---`n`"name`": `"Ada`"`n`"active`": true") { throw 'The imported formatter did not normalize YAML.' } +$merged = Merge-Yaml -InputObject @( + 'service: { image: example:v1, ports: [80] }', + 'service: { image: example:v2, ports: [443] }' +) +$mergedValue = $merged | ConvertFrom-Yaml -AsHashtable +if ($mergedValue['service']['image'] -cne 'example:v2' -or + $mergedValue['service']['ports'][0] -ne 443) { + throw 'The imported merge command did not apply later stream precedence.' +} $shared = [ordered]@{ value = 1 } $roundTrip = [ordered]@{ first = $shared; second = $shared } | ConvertTo-Yaml | From 92bf29209c4fa5c837bb27627601037ba49f61b9 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sat, 25 Jul 2026 07:15:47 +0200 Subject: [PATCH 3/8] Document layered YAML stream merging Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ examples/General.ps1 | 14 ++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/README.md b/README.md index 9b1b453..21ac612 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ The module exports: | `Export-Yaml` | Serialize values and atomically write one YAML file. | | `Format-Yaml` | Normalize YAML streams without projecting representation nodes to PowerShell values. | | `Import-Yaml` | Strictly decode and parse YAML files. | +| `Merge-Yaml` | Merge complete YAML streams without losing representation graph details. | | `Test-Yaml` | Test YAML syntax, tags, duplicate keys, and configured resource limits. | ## Parse YAML @@ -174,6 +175,47 @@ $normalized -ceq ($normalized | Format-Yaml -Indent 4) duplicate representation keys, undefined aliases, malformed tags, and resource limit violations terminate with the same classified YAML errors as parsing. +## Merge YAML streams + +`Merge-Yaml` combines two or more complete YAML streams directly through their +representation graphs. Every array element or pipeline record is one complete +stream, and every stream must contain the same positive document count. Later +streams have higher precedence, and documents merge pairwise by zero-based index. + +```powershell +$baseYaml = Get-Content -LiteralPath '.\base.yaml' -Raw +$overlayYaml = Get-Content -LiteralPath '.\overlay.yaml' -Raw +$mergedYaml = Merge-Yaml -InputObject @($baseYaml, $overlayYaml) +``` + +Compatible mappings merge recursively by structural YAML key equality. Base key +order remains stable, replacing a value retains its position, and new overlay +keys append in overlay order. Complex and tagged keys are supported. Structural +fingerprints select comparison candidates only; graph-aware equality makes the +final key decision. + +Compatible sequences use `-SequenceAction Replace`, `Append`, or `Unique`. +Unequal scalars, collection kinds, and incompatible effective tags use +`-ConflictAction Replace` or `Error`. A later YAML null uses `-NullAction +Replace` or `Ignore`; ignoring retains an existing prior node, including at a +document root. + +```powershell +$baseYaml, $environmentYaml, $secretYaml | + Merge-Yaml -SequenceAction Unique -ConflictAction Error -Indent 4 +``` + +Tags, anchors, aliases, repeated nodes, cycles, mapping order, and selected +representation nodes remain graph data. Inputs are immutable, and YAML 1.1 `<<` +merge keys remain ordinary mapping entries rather than being expanded. Output is +one deterministic string with LF line endings, explicit document starts, and no +final newline. + +The parser safety parameters and defaults match `Format-Yaml`. `-MaxNodes` +limits each parsed stream and invocation-wide cloning, equality, merge work, and +the resulting stream graph. Alias and expanded-tag budgets are also enforced on +the result. + ## Export YAML files `Export-Yaml` aggregates pipeline records like `ConvertTo-Yaml`, serializes the @@ -220,6 +262,8 @@ limit violations. Unexpected runtime failures are not suppressed. represent it. - YAML merge keys are not expanded; `<<` is ordinary mapping data under the YAML 1.2 core schema. +- YAML stream merging compares effective tags and structural representation + values without projecting through PowerShell objects. - Parsing defaults to depth 100, 100000 nodes, 1000 aliases, 1048576 decoded characters per scalar, 1024 characters per expanded tag, 65536 cumulative expanded tag characters, and 4096 digits per numeric scalar. The diff --git a/examples/General.ps1 b/examples/General.ps1 index 050afc6..dc69e5a 100644 --- a/examples/General.ps1 +++ b/examples/General.ps1 @@ -47,6 +47,20 @@ $normalizedYaml = @' '@ | Format-Yaml -Indent 4 $normalizedYaml +# Merge complete YAML streams without projecting their representation graphs. +$baseYaml = @' +service: + image: example:v1 + ports: [80] +'@ +$overlayYaml = @' +service: + image: example:v2 + ports: [443] +'@ +$mergedYaml = Merge-Yaml -InputObject @($baseYaml, $overlayYaml) +$mergedYaml + # Atomically export one file, then import it with strict decoding. $configPath = Join-Path $env:TEMP 'yaml-example.yaml' $config | Export-Yaml -Path $configPath -PassThru From fd44f0181d57d1c5494877ae4cd784ff9ab245a2 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sat, 25 Jul 2026 07:16:52 +0200 Subject: [PATCH 4/8] Harden YAML merge policy coverage Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/Merge-Yaml.Tests.ps1 | 67 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/tests/Merge-Yaml.Tests.ps1 b/tests/Merge-Yaml.Tests.ps1 index 1b677e3..ef228a4 100644 --- a/tests/Merge-Yaml.Tests.ps1 +++ b/tests/Merge-Yaml.Tests.ps1 @@ -228,6 +228,21 @@ tail: base $entry.Value['second'] | Should -Be 2 } + It 'does not merge unequal complex keys that share a candidate bucket' { + $base = @' +? [region, { port: 443 }] +: first +'@ + $overlay = @' +? [zone, { port: 443 }] +: second +'@ + $result = Merge-Yaml $base, $overlay | ConvertFrom-Yaml -AsHashtable + + $result.Count | Should -Be 2 + @($result.Values) | Should -Be @('first', 'second') + } + It 'keeps YAML 1.1 merge syntax as ordinary mapping data' { $base = @' <<: @@ -298,6 +313,7 @@ items: - one - key: value - !item tagged + - !other tagged - &other { self: *other } - added '@ @@ -305,13 +321,14 @@ items: $result = $merged | ConvertFrom-Yaml -AsHashtable $facts = Get-MergeYamlGraphFact -Yaml $merged - $result['items'].Count | Should -Be 5 - $result['items'][4] | Should -Be 'added' + $result['items'].Count | Should -Be 6 + $result['items'][5] | Should -Be 'added' [object]::ReferenceEquals( $result['items'][3], $result['items'][3]['self'] ) | Should -BeTrue @($facts.Tags) | Should -Contain '!item' + @($facts.Tags) | Should -Contain '!other' } It 'preserves overlay sharing and cycles while appending' { @@ -363,6 +380,16 @@ items: Should -BeExactly ($base | Format-Yaml) } + It 'does not apply sequence actions across incompatible effective tags' { + $failure = Get-MergeYamlFailure { + Merge-Yaml '!first [one]', '!second [two]' ` + -SequenceAction Append -ConflictAction Error + } + + $failure.Exception.Data['YamlErrorId'] | Should -BeExactly 'YamlMergeConflict' + $failure.Exception.Message | Should -Match 'sequence tag' + } + It 'applies nested null replacement and ignoring' { $base = "root:`n value: retained" $overlay = "root:`n value: null" @@ -471,6 +498,17 @@ node: &cycle $documents[1]['second'] | Should -Be 'overlay' } + It 'keeps explicit empty documents as positive document records' { + $base = "---`nvalue: base`n---" + $overlay = "---`nvalue: overlay`n---" + $merged = Merge-Yaml $base, $overlay + $documents = @($merged | ConvertFrom-Yaml -AsHashtable) + + $documents.Count | Should -Be 2 + $documents[0]['value'] | Should -Be 'overlay' + $documents[1] | Should -BeNullOrEmpty + } + It 'rejects no-document streams with their zero-based input index' -ForEach @( @{ Empty = '' } @{ Empty = "# comment only`n" } @@ -507,6 +545,31 @@ node: &cycle Should -Be 'YamlInvalidFlowCollection,Merge-Yaml' } + It 'preserves every parser resource classification' -ForEach @( + @{ Yaml = "a:`n b:`n c: value"; Parameters = @{ Depth = 2 } } + @{ Yaml = '[one, two]'; Parameters = @{ MaxNodes = 2 } } + @{ Yaml = "a: &a value`nb: *a"; Parameters = @{ MaxAliases = 0 } } + @{ Yaml = 'value: long'; Parameters = @{ MaxScalarLength = 4 } } + @{ Yaml = '!long value'; Parameters = @{ MaxTagLength = 2 } } + @{ + Yaml = "!a one`n---`n!b two" + Parameters = @{ MaxTotalTagLength = 3 } + } + @{ Yaml = '123'; Parameters = @{ MaxNumericLength = 2 } } + ) { + $parseFailure = Get-MergeYamlFailure { + $Yaml | Format-Yaml @Parameters + } + $mergeFailure = Get-MergeYamlFailure { + Merge-Yaml 'valid: true', $Yaml @Parameters + } + + $mergeFailure.Exception.Data['YamlErrorId'] | + Should -BeExactly $parseFailure.Exception.Data['YamlErrorId'] + $mergeFailure.FullyQualifiedErrorId | + Should -Be "$($parseFailure.Exception.Data['YamlErrorId']),Merge-Yaml" + } + It 'enforces clone and result node budgets' { $failure = Get-MergeYamlFailure { Merge-Yaml 'a: 1', 'b: 2' -MaxNodes 3 From 8acf4f6623876e898681d63e4248dd616bdfa37e Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sat, 25 Jul 2026 07:34:44 +0200 Subject: [PATCH 5/8] Link generated YAML merge help Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/functions/public/Merge-Yaml.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/functions/public/Merge-Yaml.ps1 b/src/functions/public/Merge-Yaml.ps1 index dabccc4..0e8e135 100644 --- a/src/functions/public/Merge-Yaml.ps1 +++ b/src/functions/public/Merge-Yaml.ps1 @@ -95,7 +95,7 @@ function Merge-Yaml { YAML 1.1 merge keys are ordinary mapping data and are never expanded. .LINK - Format-Yaml + https://github.com/PSModule/Yaml#format-yaml-streams #> [OutputType([string])] [CmdletBinding()] From 47a3e9dbfb431842b63325b8ff2c0d9e6696c95e Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sat, 25 Jul 2026 07:45:17 +0200 Subject: [PATCH 6/8] Isolate fresh-process YAML merge smoke Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/Packaging.Tests.ps1 | 51 ++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/tests/Packaging.Tests.ps1 b/tests/Packaging.Tests.ps1 index 6e62607..2824f17 100644 --- a/tests/Packaging.Tests.ps1 +++ b/tests/Packaging.Tests.ps1 @@ -177,15 +177,6 @@ $formatted = '{name: Ada, active: true}' | Format-Yaml if ($formatted -cne "---`n`"name`": `"Ada`"`n`"active`": true") { throw 'The imported formatter did not normalize YAML.' } -$merged = Merge-Yaml -InputObject @( - 'service: { image: example:v1, ports: [80] }', - 'service: { image: example:v2, ports: [443] }' -) -$mergedValue = $merged | ConvertFrom-Yaml -AsHashtable -if ($mergedValue['service']['image'] -cne 'example:v2' -or - $mergedValue['service']['ports'][0] -ne 443) { - throw 'The imported merge command did not apply later stream precedence.' -} $shared = [ordered]@{ value = 1 } $roundTrip = [ordered]@{ first = $shared; second = $shared } | ConvertTo-Yaml | @@ -217,6 +208,7 @@ if (-not (Test-Yaml -Yaml $deepYaml -Depth 128 -MaxNodes 300)) { '@.Replace('__MANIFEST__', $artifactManifestPath.Replace("'", "''")) $output = @(& pwsh -NoLogo -NoProfile -Command $script) + $LASTEXITCODE | Should -Be 0 $runtime = $output | Where-Object { $_ -like 'powershell-runtime=*' } | Select-Object -Last 1 @@ -228,6 +220,47 @@ if (-not (Test-Yaml -Yaml $deepYaml -Depth 128 -MaxNodes 300)) { ([version] $runtimeMatch.Groups['Version'].Value) -lt [version] '7.6' | Should -BeFalse Write-Information -MessageData $runtime -InformationAction Continue + } + + It 'merges complete streams in a fresh PowerShell 7.6 Core process' ` + -Skip:($skipArtifactTests -or $null -eq (Get-Command pwsh -ErrorAction SilentlyContinue)) { + $script = @' +$ErrorActionPreference = 'Stop' +$ps = $PSVersionTable.PSVersion +if ($ps -lt [version] '7.6') { + throw "Expected PowerShell 7.6 or newer but got $ps." +} +if ($PSVersionTable.PSEdition -cne 'Core') { + throw "Expected PowerShell Core but got $($PSVersionTable.PSEdition)." +} +Import-Module -Name '__MANIFEST__' -Force +$merged = Merge-Yaml -InputObject @( + 'service: { image: example:v1, ports: [80] }', + 'service: { image: example:v2, ports: [443] }' +) +if ($merged -match "`r" -or $merged.EndsWith("`n", [System.StringComparison]::Ordinal)) { + throw 'The imported merge command did not normalize its output contract.' +} +$mergedValue = $merged | ConvertFrom-Yaml -AsHashtable +if ($mergedValue['service']['image'] -cne 'example:v2' -or + $mergedValue['service']['ports'][0] -ne 443) { + throw 'The imported merge command did not apply later stream precedence.' +} +"merge-runtime=$ps;edition=$($PSVersionTable.PSEdition)" +'@.Replace('__MANIFEST__', $artifactManifestPath.Replace("'", "''")) + + $output = @(& pwsh -NoLogo -NoProfile -Command $script) $LASTEXITCODE | Should -Be 0 + $runtime = $output | Where-Object { $_ -like 'merge-runtime=*' } | + Select-Object -Last 1 + + $runtimeMatch = [regex]::Match( + [string] $runtime, + '^merge-runtime=(?\d+(?:\.\d+){1,3});edition=Core$' + ) + $runtimeMatch.Success | Should -BeTrue + ([version] $runtimeMatch.Groups['Version'].Value) -lt [version] '7.6' | + Should -BeFalse + Write-Information -MessageData $runtime -InformationAction Continue } } From 48adb37e7e4eddd4ac5b84f85fa646365f7fa4c8 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sat, 25 Jul 2026 08:58:56 +0200 Subject: [PATCH 7/8] Bound YAML merge graph indexing work Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../private/Add-YamlMergeIndexCandidate.ps1 | 44 +++ src/functions/private/Add-YamlMergeWork.ps1 | 36 +++ .../private/Find-YamlMergeIndexMatch.ps1 | 42 +++ .../private/Get-YamlMergeFingerprint.ps1 | 252 +++++++++++++++++- src/functions/private/Get-YamlMergeIndex.ps1 | 62 +++++ .../private/Merge-YamlRepresentationNode.ps1 | 119 ++++----- .../private/Set-YamlMergeNodeChanged.ps1 | 34 +++ .../private/Test-YamlMergeNodeEqual.ps1 | 117 ++++++-- src/functions/public/Merge-Yaml.ps1 | 47 +++- tests/Merge-Yaml.Tests.ps1 | 250 ++++++++++++++++- 10 files changed, 884 insertions(+), 119 deletions(-) create mode 100644 src/functions/private/Add-YamlMergeIndexCandidate.ps1 create mode 100644 src/functions/private/Add-YamlMergeWork.ps1 create mode 100644 src/functions/private/Find-YamlMergeIndexMatch.ps1 create mode 100644 src/functions/private/Get-YamlMergeIndex.ps1 create mode 100644 src/functions/private/Set-YamlMergeNodeChanged.ps1 diff --git a/src/functions/private/Add-YamlMergeIndexCandidate.ps1 b/src/functions/private/Add-YamlMergeIndexCandidate.ps1 new file mode 100644 index 0000000..4c4e057 --- /dev/null +++ b/src/functions/private/Add-YamlMergeIndexCandidate.ps1 @@ -0,0 +1,44 @@ +function Add-YamlMergeIndexCandidate { + <# + .SYNOPSIS + Adds one mapping key or sequence item to a structural candidate index. + #> + [CmdletBinding()] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Index, + + [Parameter(Mandatory)] + [pscustomobject] $Candidate, + + [Parameter(Mandatory)] + [pscustomobject] $Context + ) + + $node = if ($Index.Kind -eq 'Mapping') { + $Candidate.Key + } else { + $Candidate + } + Add-YamlMergeWork -State $Context.WorkState -Node $node ` + -Operation 'index candidate identity' + $effective = Get-YamlMergeNode -Node $node + if (-not $Index.IndexedEffectiveIds.Add($effective.Id)) { + return + } + + $fingerprint = Get-YamlMergeFingerprint -Node $node -State $Context.EqualityState ` + -Cache $Index.FingerprintCache -CandidateIndex $Index + + Add-YamlMergeWork -State $Context.WorkState -Node $node -Operation 'index bucket visit' + $bucket = $null + if (-not $Index.Buckets.TryGetValue($fingerprint, [ref] $bucket)) { + $bucket = [pscustomobject]@{ + Candidates = [System.Collections.Generic.List[object]]::new() + } + $Index.Buckets[$fingerprint] = $bucket + } + + Add-YamlMergeWork -State $Context.WorkState -Node $node -Operation 'index candidate visit' + $bucket.Candidates.Add($Candidate) +} diff --git a/src/functions/private/Add-YamlMergeWork.ps1 b/src/functions/private/Add-YamlMergeWork.ps1 new file mode 100644 index 0000000..1088e18 --- /dev/null +++ b/src/functions/private/Add-YamlMergeWork.ps1 @@ -0,0 +1,36 @@ +function Add-YamlMergeWork { + <# + .SYNOPSIS + Charges one or more deterministic operations to the YAML merge work budget. + #> + [CmdletBinding()] + param ( + [Parameter(Mandatory)] + [pscustomobject] $State, + + [Parameter()] + [ValidateRange(1, 2147483647)] + [int] $Count = 1, + + [Parameter(Mandatory)] + [string] $Operation, + + [Parameter()] + [AllowNull()] + [pscustomobject] $Node + ) + + $State.Count = [long] $State.Count + $Count + if ($State.Count -le $State.MaxNodes) { + return + } + + $exception = New-YamlMergeException -Node $Node -ErrorId 'YamlMergeWorkLimitExceeded' ` + -Message ( + "YAML merge operation '$Operation' exceeded the configured invocation work limit " + + "of $($State.MaxNodes) operations." + ) + $exception.Data['YamlMergeWorkCount'] = $State.Count + $exception.Data['YamlMergeWorkLimit'] = $State.MaxNodes + throw $exception +} diff --git a/src/functions/private/Find-YamlMergeIndexMatch.ps1 b/src/functions/private/Find-YamlMergeIndexMatch.ps1 new file mode 100644 index 0000000..a8df6a3 --- /dev/null +++ b/src/functions/private/Find-YamlMergeIndexMatch.ps1 @@ -0,0 +1,42 @@ +function Find-YamlMergeIndexMatch { + <# + .SYNOPSIS + Finds a collision-safe structural match in a YAML merge candidate index. + #> + [CmdletBinding()] + [OutputType([pscustomobject])] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Index, + + [Parameter(Mandatory)] + [pscustomobject] $Node, + + [Parameter(Mandatory)] + [pscustomobject] $Context + ) + + $fingerprint = Get-YamlMergeFingerprint -Node $Node -State $Context.EqualityState ` + -Cache $Context.OverlayFingerprintCache + Add-YamlMergeWork -State $Context.WorkState -Node $Node -Operation 'index bucket lookup' + + $bucket = $null + if (-not $Index.Buckets.TryGetValue($fingerprint, [ref] $bucket)) { + return + } + + foreach ($candidate in $bucket.Candidates) { + $candidateNode = if ($Index.Kind -eq 'Mapping') { + $candidate.Key + } else { + $candidate + } + Add-YamlMergeWork -State $Context.WorkState -Node $Node ` + -Operation 'index candidate comparison' + if (Test-YamlMergeNodeEqual -Node $candidateNode -OtherNode $Node ` + -State $Context.EqualityState) { + Write-Output -InputObject $candidate -NoEnumerate + return + } + } +} diff --git a/src/functions/private/Get-YamlMergeFingerprint.ps1 b/src/functions/private/Get-YamlMergeFingerprint.ps1 index e726e79..36f75a9 100644 --- a/src/functions/private/Get-YamlMergeFingerprint.ps1 +++ b/src/functions/private/Get-YamlMergeFingerprint.ps1 @@ -1,7 +1,7 @@ function Get-YamlMergeFingerprint { <# .SYNOPSIS - Creates a deterministic candidate index for structural merge comparisons. + Creates a deterministic structural candidate index for merge comparisons. #> [CmdletBinding()] [OutputType([string])] @@ -10,21 +10,247 @@ function Get-YamlMergeFingerprint { [pscustomobject] $Node, [Parameter(Mandatory)] - [pscustomobject] $State + [pscustomobject] $State, + + [Parameter()] + [AllowNull()] + [System.Collections.Generic.Dictionary[int, string]] $Cache, + + [Parameter()] + [AllowNull()] + [pscustomobject] $CandidateIndex ) - $effective = Get-YamlMergeNode -Node $Node - $tag = Get-YamlMergeNodeTag -Node $effective - if ($effective.Kind -eq 'Scalar') { - $resolved = (Resolve-YamlScalar -Node $effective).Value - $value = Get-YamlScalarFingerprint -Value $resolved -Hasher $State.FingerprintHasher - return 'scalar:{0}:{1}:{2}' -f $tag.Length, $tag, $value + if ($null -eq $Cache) { + $Cache = [System.Collections.Generic.Dictionary[int, string]]::new() } + $active = [System.Collections.Generic.HashSet[int]]::new() + $root = [pscustomobject]@{ Value = ''; IsContextual = $false } + $stack = [System.Collections.Generic.Stack[object]]::new() + $stack.Push([pscustomobject]@{ + Node = $Node + Holder = $root + State = 'Start' + Index = 0 + Parts = $null + Child = $null + KeyHash = '' + Accumulator = $null + IsContextual = $false + }) + + while ($stack.Count -gt 0) { + $frame = $stack.Peek() + if ($frame.State -eq 'Start') { + $effective = $frame.Node + while ($effective.Kind -eq 'Alias') { + Add-YamlMergeWork -State $State.WorkState -Node $effective ` + -Operation 'fingerprint alias traversal' + $effective = $effective.Target + } + Add-YamlMergeWork -State $State.WorkState -Node $effective ` + -Operation 'fingerprint node traversal' + $frame.Node = $effective + + if ($null -ne $CandidateIndex -and + $CandidateIndex.Dependencies.Add($effective.Id)) { + $dependents = $null + if (-not $State.IndexDependents.TryGetValue( + $effective.Id, + [ref] $dependents + )) { + $dependents = [System.Collections.Generic.List[object]]::new() + $State.IndexDependents[$effective.Id] = $dependents + } + $dependents.Add($CandidateIndex) + } + + $cached = '' + if ($Cache.TryGetValue($effective.Id, [ref] $cached)) { + $frame.Holder.Value = $cached + [void] $stack.Pop() + continue + } - $count = if ($effective.Kind -eq 'Sequence') { - $effective.Items.Count - } else { - $effective.Entries.Count + if (-not $active.Add($effective.Id)) { + $tag = Get-YamlMergeNodeTag -Node $effective + $count = if ($effective.Kind -eq 'Sequence') { + $effective.Items.Count + } elseif ($effective.Kind -eq 'Mapping') { + $effective.Entries.Count + } else { + 0 + } + $frame.Holder.Value = Get-YamlFingerprintHash -Value ( + 'cycle:{0}:{1}:{2}:{3}' -f @( + $effective.Kind.ToLowerInvariant(), + $tag.Length, + $tag, + $count + ) + ) -Hasher $State.FingerprintHasher + $frame.Holder.IsContextual = $true + [void] $stack.Pop() + continue + } + + if ($effective.Kind -eq 'Scalar') { + $resolved = (Resolve-YamlScalar -Node $effective).Value + $tag = Get-YamlEffectiveTag -Node $effective -Value $resolved + $value = Get-YamlScalarFingerprint -Value $resolved ` + -Hasher $State.FingerprintHasher + $fingerprint = Get-YamlFingerprintHash -Value ( + 'scalar:{0}:{1}:{2}' -f $tag.Length, $tag, $value + ) -Hasher $State.FingerprintHasher + $Cache[$effective.Id] = $fingerprint + [void] $active.Remove($effective.Id) + $frame.Holder.Value = $fingerprint + [void] $stack.Pop() + continue + } + + if ($effective.Kind -eq 'Sequence') { + $frame.Parts = [System.Collections.Generic.List[string]]::new() + $frame.State = 'Sequence' + } else { + $frame.Accumulator = [int[]]::new(32) + $frame.State = 'MappingKey' + } + continue + } + + if ($frame.State -eq 'Sequence') { + if ($frame.Index -ge $frame.Node.Items.Count) { + $tag = Get-YamlEffectiveTag -Node $frame.Node -Value $null + $canonical = if ($frame.IsContextual) { + 'cyclic-sequence:{0}:{1}:{2}' -f @( + $tag.Length, + $tag, + $frame.Node.Items.Count + ) + } else { + 'sequence:{0}:{1}:{2}' -f $tag.Length, $tag, ($frame.Parts -join '|') + } + $fingerprint = Get-YamlFingerprintHash -Value $canonical ` + -Hasher $State.FingerprintHasher + if (-not $frame.IsContextual) { + $Cache[$frame.Node.Id] = $fingerprint + } + [void] $active.Remove($frame.Node.Id) + $frame.Holder.Value = $fingerprint + $frame.Holder.IsContextual = $frame.IsContextual + [void] $stack.Pop() + continue + } + Add-YamlMergeWork -State $State.WorkState -Node $frame.Node ` + -Operation 'fingerprint sequence entry' + $frame.Child = [pscustomobject]@{ Value = ''; IsContextual = $false } + $frame.State = 'SequenceValue' + $stack.Push([pscustomobject]@{ + Node = $frame.Node.Items[$frame.Index] + Holder = $frame.Child + State = 'Start' + Index = 0 + Parts = $null + Child = $null + KeyHash = '' + Accumulator = $null + IsContextual = $false + }) + continue + } + if ($frame.State -eq 'SequenceValue') { + $frame.Parts.Add($frame.Child.Value) + $frame.IsContextual = $frame.IsContextual -or $frame.Child.IsContextual + $frame.Index++ + $frame.State = 'Sequence' + continue + } + + if ($frame.State -eq 'MappingKey') { + if ($frame.Index -ge $frame.Node.Entries.Count) { + $tag = Get-YamlEffectiveTag -Node $frame.Node -Value $null + $canonical = if ($frame.IsContextual) { + 'cyclic-mapping:{0}:{1}:{2}' -f @( + $tag.Length, + $tag, + $frame.Node.Entries.Count + ) + } else { + $aggregate = [byte[]]::new($frame.Accumulator.Count) + for ($index = 0; $index -lt $aggregate.Count; $index++) { + $aggregate[$index] = [byte] $frame.Accumulator[$index] + } + 'mapping:{0}:{1}:{2}:{3}' -f @( + $tag.Length, + $tag, + $frame.Node.Entries.Count, + [System.Convert]::ToBase64String($aggregate) + ) + } + $fingerprint = Get-YamlFingerprintHash -Value $canonical ` + -Hasher $State.FingerprintHasher + if (-not $frame.IsContextual) { + $Cache[$frame.Node.Id] = $fingerprint + } + [void] $active.Remove($frame.Node.Id) + $frame.Holder.Value = $fingerprint + $frame.Holder.IsContextual = $frame.IsContextual + [void] $stack.Pop() + continue + } + Add-YamlMergeWork -State $State.WorkState -Node $frame.Node ` + -Operation 'fingerprint mapping entry' + $frame.Child = [pscustomobject]@{ Value = ''; IsContextual = $false } + $frame.State = 'MappingKeyValue' + $stack.Push([pscustomobject]@{ + Node = $frame.Node.Entries[$frame.Index].Key + Holder = $frame.Child + State = 'Start' + Index = 0 + Parts = $null + Child = $null + KeyHash = '' + Accumulator = $null + IsContextual = $false + }) + continue + } + if ($frame.State -eq 'MappingKeyValue') { + $frame.KeyHash = $frame.Child.Value + $frame.IsContextual = $frame.IsContextual -or $frame.Child.IsContextual + $frame.Child = [pscustomobject]@{ Value = ''; IsContextual = $false } + $frame.State = 'MappingValue' + $stack.Push([pscustomobject]@{ + Node = $frame.Node.Entries[$frame.Index].Value + Holder = $frame.Child + State = 'Start' + Index = 0 + Parts = $null + Child = $null + KeyHash = '' + Accumulator = $null + IsContextual = $false + }) + continue + } + if ($frame.State -eq 'MappingValue') { + Add-YamlMergeWork -State $State.WorkState -Node $frame.Node ` + -Operation 'fingerprint mapping combination' + $entryHash = Get-YamlFingerprintHash -Value ( + "entry:$($frame.KeyHash)=$($frame.Child.Value)" + ) -Hasher $State.FingerprintHasher + $entryBytes = [System.Convert]::FromBase64String($entryHash) + for ($index = 0; $index -lt $entryBytes.Count; $index++) { + $frame.Accumulator[$index] = ( + $frame.Accumulator[$index] + $entryBytes[$index] + ) -band 0xff + } + $frame.IsContextual = $frame.IsContextual -or $frame.Child.IsContextual + $frame.Index++ + $frame.State = 'MappingKey' + } } - return '{0}:{1}:{2}:{3}' -f $effective.Kind.ToLowerInvariant(), $tag.Length, $tag, $count + + $root.Value } diff --git a/src/functions/private/Get-YamlMergeIndex.ps1 b/src/functions/private/Get-YamlMergeIndex.ps1 new file mode 100644 index 0000000..33912df --- /dev/null +++ b/src/functions/private/Get-YamlMergeIndex.ps1 @@ -0,0 +1,62 @@ +function Get-YamlMergeIndex { + <# + .SYNOPSIS + Gets or rebuilds a mutation-aware structural candidate index. + #> + [CmdletBinding()] + [OutputType([pscustomobject])] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Node, + + [Parameter(Mandatory)] + [ValidateSet('Mapping', 'Sequence')] + [string] $Kind, + + [Parameter(Mandatory)] + [pscustomobject] $Context + ) + + $effective = Get-YamlMergeNode -Node $Node + $cache = if ($Kind -eq 'Mapping') { + $Context.MappingIndexes + } else { + $Context.SequenceIndexes + } + + $index = $null + if (-not $cache.TryGetValue($effective.Id, [ref] $index)) { + $index = [pscustomobject]@{ + Kind = $Kind + IsValid = $false + Buckets = [System.Collections.Generic.Dictionary[string, object]]::new( + [System.StringComparer]::Ordinal + ) + Dependencies = [System.Collections.Generic.HashSet[int]]::new() + FingerprintCache = [System.Collections.Generic.Dictionary[int, string]]::new() + IndexedEffectiveIds = [System.Collections.Generic.HashSet[int]]::new() + } + $cache[$effective.Id] = $index + } + + if ($index.IsValid) { + Write-Output -InputObject $index -NoEnumerate + return + } + + Add-YamlMergeWork -State $Context.WorkState -Node $effective -Operation 'index rebuild' + $index.Buckets.Clear() + $index.FingerprintCache.Clear() + $index.IndexedEffectiveIds.Clear() + $candidates = if ($Kind -eq 'Mapping') { + $effective.Entries + } else { + $effective.Items + } + foreach ($candidate in $candidates) { + Add-YamlMergeIndexCandidate -Index $index -Candidate $candidate -Context $Context + } + $index.IsValid = $true + + Write-Output -InputObject $index -NoEnumerate +} diff --git a/src/functions/private/Merge-YamlRepresentationNode.ps1 b/src/functions/private/Merge-YamlRepresentationNode.ps1 index 39779bd..3ce15cc 100644 --- a/src/functions/private/Merge-YamlRepresentationNode.ps1 +++ b/src/functions/private/Merge-YamlRepresentationNode.ps1 @@ -35,13 +35,8 @@ function Merge-YamlRepresentationNode { [pscustomobject] $Context ) - $Context.WorkState.MergeCount++ - if ($Context.WorkState.MergeCount -gt $Context.WorkState.MaxNodes) { - throw (New-YamlMergeException -Node $OverlayNode -ErrorId 'YamlMergeNodeLimitExceeded' -Message ( - "Merging input index $($Context.InputIndex) document index $($Context.DocumentIndex) " + - "exceeded the configured work limit of $($Context.WorkState.MaxNodes) nodes." - )) - } + Add-YamlMergeWork -State $Context.WorkState -Node $OverlayNode ` + -Operation 'representation node merge' $base = Get-YamlMergeNode -Node $BaseNode $overlay = Get-YamlMergeNode -Node $OverlayNode @@ -52,16 +47,6 @@ function Merge-YamlRepresentationNode { return } - if (Test-YamlMergeNodeEqual -Node $base -OtherNode $overlay ` - -State $Context.EqualityState) { - if ($overlay.Kind -ne 'Scalar' -and - -not $Context.CloneCache.ContainsKey($overlay.Id)) { - $Context.CloneCache[$overlay.Id] = $base - } - Write-Output -InputObject $BaseNode -NoEnumerate - return - } - $baseTag = Get-YamlMergeNodeTag -Node $base $isCompatible = $base.Kind -ceq $overlay.Kind -and $baseTag -ceq $overlayTag if (-not $isCompatible) { @@ -77,6 +62,11 @@ function Merge-YamlRepresentationNode { } if ($base.Kind -eq 'Scalar') { + if (Test-YamlMergeNodeEqual -Node $base -OtherNode $overlay ` + -State $Context.EqualityState) { + Write-Output -InputObject $BaseNode -NoEnumerate + return + } if ($ConflictAction -eq 'Error') { throw (New-YamlMergeException -Node $overlay -ErrorId 'YamlMergeConflict' -Message ( "YAML merge conflict at $Path in input index $($Context.InputIndex), " + @@ -88,6 +78,14 @@ function Merge-YamlRepresentationNode { } if ($base.Kind -eq 'Sequence' -and $SequenceAction -eq 'Replace') { + if (Test-YamlMergeNodeEqual -Node $base -OtherNode $overlay ` + -State $Context.EqualityState) { + if (-not $Context.CloneCache.ContainsKey($overlay.Id)) { + $Context.CloneCache[$overlay.Id] = $base + } + Write-Output -InputObject $BaseNode -NoEnumerate + return + } return Copy-YamlMergeNode -Node $OverlayNode -Cache $Context.CloneCache ` -State $Context.CloneState } @@ -100,88 +98,71 @@ function Merge-YamlRepresentationNode { Write-Output -InputObject $Context.CloneCache[$overlay.Id] -NoEnumerate return } + + if (Test-YamlMergeNodeEqual -Node $base -OtherNode $overlay ` + -State $Context.EqualityState) { + $Context.CloneCache[$overlay.Id] = $base + Write-Output -InputObject $BaseNode -NoEnumerate + return + } $Context.CloneCache[$overlay.Id] = $base if ($base.Kind -eq 'Sequence') { if ($SequenceAction -eq 'Append') { + $changed = $false foreach ($item in $overlay.Items) { $copy = Copy-YamlMergeNode -Node $item -Cache $Context.CloneCache ` -State $Context.CloneState $base.Items.Add($copy) + $changed = $true + } + if ($changed) { + Set-YamlMergeNodeChanged -Node $base -Context $Context } Write-Output -InputObject $BaseNode -NoEnumerate return } - $retained = [System.Collections.Generic.Dictionary[string, object]]::new( - [System.StringComparer]::Ordinal - ) - foreach ($item in $base.Items) { - $fingerprint = Get-YamlMergeFingerprint -Node $item -State $Context.EqualityState - if (-not $retained.ContainsKey($fingerprint)) { - $retained[$fingerprint] = [System.Collections.Generic.List[object]]::new() - } - $retained[$fingerprint].Add($item) - } + $overlayItems = [System.Collections.Generic.HashSet[int]]::new() foreach ($item in $overlay.Items) { - $fingerprint = Get-YamlMergeFingerprint -Node $item -State $Context.EqualityState - $exists = $false - if ($retained.ContainsKey($fingerprint)) { - foreach ($candidate in $retained[$fingerprint]) { - if (Test-YamlMergeNodeEqual -Node $candidate -OtherNode $item ` - -State $Context.EqualityState) { - $exists = $true - break - } - } + Add-YamlMergeWork -State $Context.WorkState -Node $item ` + -Operation 'unique overlay candidate identity' + $effectiveItem = Get-YamlMergeNode -Node $item + if (-not $overlayItems.Add($effectiveItem.Id)) { + continue } - if ($exists) { + $retained = Get-YamlMergeIndex -Node $base -Kind Sequence -Context $Context + $match = Find-YamlMergeIndexMatch -Index $retained -Node $item -Context $Context + if ($null -ne $match) { continue } $copy = Copy-YamlMergeNode -Node $item -Cache $Context.CloneCache ` -State $Context.CloneState $base.Items.Add($copy) - if (-not $retained.ContainsKey($fingerprint)) { - $retained[$fingerprint] = [System.Collections.Generic.List[object]]::new() - } - $retained[$fingerprint].Add($copy) + Add-YamlMergeIndexCandidate -Index $retained -Candidate $copy -Context $Context + Set-YamlMergeNodeChanged -Node $base -Context $Context } Write-Output -InputObject $BaseNode -NoEnumerate return } - $entries = [System.Collections.Generic.Dictionary[string, object]]::new( - [System.StringComparer]::Ordinal - ) - foreach ($entry in $base.Entries) { - $fingerprint = Get-YamlMergeFingerprint -Node $entry.Key -State $Context.EqualityState - if (-not $entries.ContainsKey($fingerprint)) { - $entries[$fingerprint] = [System.Collections.Generic.List[object]]::new() - } - $entries[$fingerprint].Add($entry) - } - for ($index = 0; $index -lt $overlay.Entries.Count; $index++) { $overlayEntry = $overlay.Entries[$index] - $fingerprint = Get-YamlMergeFingerprint -Node $overlayEntry.Key ` - -State $Context.EqualityState - $match = $null - if ($entries.ContainsKey($fingerprint)) { - foreach ($candidate in $entries[$fingerprint]) { - if (Test-YamlMergeNodeEqual -Node $candidate.Key -OtherNode $overlayEntry.Key ` - -State $Context.EqualityState) { - $match = $candidate - break - } - } - } + $entries = Get-YamlMergeIndex -Node $base -Kind Mapping -Context $Context + $match = Find-YamlMergeIndexMatch -Index $entries -Node $overlayEntry.Key ` + -Context $Context if ($null -ne $match) { $childPath = Get-YamlMergePath -Parent $Path -Key $overlayEntry.Key -Index $index - $match.Value = Merge-YamlRepresentationNode -BaseNode $match.Value ` + $previous = $match.Value + $merged = Merge-YamlRepresentationNode -BaseNode $previous ` -OverlayNode $overlayEntry.Value -SequenceAction $SequenceAction ` -ConflictAction $ConflictAction -NullAction $NullAction -Path $childPath ` -Context $Context + $match.Value = $merged + if (-not [object]::ReferenceEquals($previous, $merged)) { + Set-YamlMergeNodeChanged -Node $base -Context $Context + } continue } @@ -191,10 +172,8 @@ function Merge-YamlRepresentationNode { -State $Context.CloneState $newEntry = [pscustomobject]@{ Key = $keyCopy; Value = $valueCopy } $base.Entries.Add($newEntry) - if (-not $entries.ContainsKey($fingerprint)) { - $entries[$fingerprint] = [System.Collections.Generic.List[object]]::new() - } - $entries[$fingerprint].Add($newEntry) + Add-YamlMergeIndexCandidate -Index $entries -Candidate $newEntry -Context $Context + Set-YamlMergeNodeChanged -Node $base -Context $Context } Write-Output -InputObject $BaseNode -NoEnumerate diff --git a/src/functions/private/Set-YamlMergeNodeChanged.ps1 b/src/functions/private/Set-YamlMergeNodeChanged.ps1 new file mode 100644 index 0000000..32bb339 --- /dev/null +++ b/src/functions/private/Set-YamlMergeNodeChanged.ps1 @@ -0,0 +1,34 @@ +function Set-YamlMergeNodeChanged { + <# + .SYNOPSIS + Invalidates indexes and equality results affected by a graph mutation. + #> + [Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSUseShouldProcessForStateChangingFunctions', '', + Justification = 'Updates only isolated in-memory merge bookkeeping.' + )] + [CmdletBinding()] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Node, + + [Parameter(Mandatory)] + [pscustomobject] $Context + ) + + $effective = Get-YamlMergeNode -Node $Node + Add-YamlMergeWork -State $Context.WorkState -Node $effective ` + -Operation 'mutation invalidation' + $Context.MutationState.Version = [long] $Context.MutationState.Version + 1 + $Context.EqualityState.Cache.Clear() + + $indexes = $null + if (-not $Context.IndexDependents.TryGetValue($effective.Id, [ref] $indexes)) { + return + } + foreach ($index in $indexes) { + Add-YamlMergeWork -State $Context.WorkState -Node $effective ` + -Operation 'dependent index invalidation' + $index.IsValid = $false + } +} diff --git a/src/functions/private/Test-YamlMergeNodeEqual.ps1 b/src/functions/private/Test-YamlMergeNodeEqual.ps1 index 2c8a3d2..3eea333 100644 --- a/src/functions/private/Test-YamlMergeNodeEqual.ps1 +++ b/src/functions/private/Test-YamlMergeNodeEqual.ps1 @@ -13,9 +13,39 @@ function Test-YamlMergeNodeEqual { [pscustomobject] $OtherNode, [Parameter(Mandatory)] - [pscustomobject] $State + [pscustomobject] $State, + + [Parameter()] + [AllowNull()] + [System.Collections.Generic.Dictionary[int, string]] $LeftFingerprintCache, + + [Parameter()] + [AllowNull()] + [System.Collections.Generic.Dictionary[int, string]] $RightFingerprintCache ) + if ($null -eq $LeftFingerprintCache) { + $LeftFingerprintCache = [System.Collections.Generic.Dictionary[int, string]]::new() + } + if ($null -eq $RightFingerprintCache) { + $RightFingerprintCache = [System.Collections.Generic.Dictionary[int, string]]::new() + } + + $rootLeft = Get-YamlMergeNode -Node $Node + $rootRight = Get-YamlMergeNode -Node $OtherNode + $cacheKey = '{0}:{1}:{2}:{3}' -f @( + $State.MutationState.Version, + $State.InputIndex, + $rootLeft.Id, + $rootRight.Id + ) + Add-YamlMergeWork -State $State.WorkState -Node $rootRight ` + -Operation 'equality cache lookup' + $cached = $false + if ($State.Cache.TryGetValue($cacheKey, [ref] $cached)) { + return $cached + } + $seen = [System.Collections.Generic.HashSet[string]]::new( [System.StringComparer]::Ordinal ) @@ -24,26 +54,34 @@ function Test-YamlMergeNodeEqual { while ($pending.Count -gt 0) { $pair = $pending.Pop() - $left = Get-YamlMergeNode -Node $pair.Left - $right = Get-YamlMergeNode -Node $pair.Right + $left = $pair.Left + while ($left.Kind -eq 'Alias') { + Add-YamlMergeWork -State $State.WorkState -Node $left ` + -Operation 'equality alias traversal' + $left = $left.Target + } + $right = $pair.Right + while ($right.Kind -eq 'Alias') { + Add-YamlMergeWork -State $State.WorkState -Node $right ` + -Operation 'equality alias traversal' + $right = $right.Target + } + Add-YamlMergeWork -State $State.WorkState -Node $right ` + -Operation 'equality pair comparison' + $pairId = '{0}:{1}' -f $left.Id, $right.Id if (-not $seen.Add($pairId)) { continue } - $State.EqualityCount++ - if ($State.EqualityCount -gt $State.MaxNodes) { - throw (New-YamlMergeException -Node $right -ErrorId 'YamlMergeEqualityLimitExceeded' -Message ( - "YAML structural equality exceeded the configured limit of $($State.MaxNodes) node pairs." - )) - } - if ($left.Kind -cne $right.Kind) { + $State.Cache[$cacheKey] = $false return $false } $leftTag = Get-YamlMergeNodeTag -Node $left $rightTag = Get-YamlMergeNodeTag -Node $right if ($leftTag -cne $rightTag) { + $State.Cache[$cacheKey] = $false return $false } @@ -52,6 +90,7 @@ function Test-YamlMergeNodeEqual { $rightValue = (Resolve-YamlScalar -Node $right).Value if ($null -eq $leftValue -or $null -eq $rightValue) { if ($null -ne $leftValue -or $null -ne $rightValue) { + $State.Cache[$cacheKey] = $false return $false } continue @@ -59,10 +98,12 @@ function Test-YamlMergeNodeEqual { if ($leftValue -is [byte[]] -or $rightValue -is [byte[]]) { if ($leftValue -isnot [byte[]] -or $rightValue -isnot [byte[]] -or $leftValue.Count -ne $rightValue.Count) { + $State.Cache[$cacheKey] = $false return $false } for ($index = 0; $index -lt $leftValue.Count; $index++) { if ($leftValue[$index] -ne $rightValue[$index]) { + $State.Cache[$cacheKey] = $false return $false } } @@ -85,6 +126,7 @@ function Test-YamlMergeNodeEqual { $rightValue.Ticks } if ($leftTicks -ne $rightTicks) { + $State.Cache[$cacheKey] = $false return $false } continue @@ -95,18 +137,21 @@ function Test-YamlMergeNodeEqual { $leftNumber = Get-YamlNormalizedFloat -Value $leftValue $rightNumber = Get-YamlNormalizedFloat -Value $rightValue if ($leftNumber -cne $rightNumber) { + $State.Cache[$cacheKey] = $false return $false } continue } if ($leftValue -is [string] -and $rightValue -is [string]) { if (-not $leftValue.Equals($rightValue, [System.StringComparison]::Ordinal)) { + $State.Cache[$cacheKey] = $false return $false } continue } if ($leftValue -is [bool] -and $rightValue -is [bool]) { if ($leftValue -ne $rightValue) { + $State.Cache[$cacheKey] = $false return $false } continue @@ -114,6 +159,7 @@ function Test-YamlMergeNodeEqual { $leftText = $leftValue.ToString([System.Globalization.CultureInfo]::InvariantCulture) $rightText = $rightValue.ToString([System.Globalization.CultureInfo]::InvariantCulture) if ($leftText -cne $rightText) { + $State.Cache[$cacheKey] = $false return $false } continue @@ -121,6 +167,7 @@ function Test-YamlMergeNodeEqual { if ($left.Kind -eq 'Sequence') { if ($left.Items.Count -ne $right.Items.Count) { + $State.Cache[$cacheKey] = $false return $false } for ($index = $left.Items.Count - 1; $index -ge 0; $index--) { @@ -133,39 +180,70 @@ function Test-YamlMergeNodeEqual { } if ($left.Entries.Count -ne $right.Entries.Count) { + $State.Cache[$cacheKey] = $false return $false } + Add-YamlMergeWork -State $State.WorkState -Node $right ` + -Operation 'equality mapping index build' $rightEntries = [System.Collections.Generic.Dictionary[string, object]]::new( [System.StringComparer]::Ordinal ) + $rightIndexedIds = [System.Collections.Generic.HashSet[int]]::new() for ($index = 0; $index -lt $right.Entries.Count; $index++) { - $fingerprint = Get-YamlMergeFingerprint -Node $right.Entries[$index].Key -State $State - if (-not $rightEntries.ContainsKey($fingerprint)) { - $rightEntries[$fingerprint] = [System.Collections.Generic.List[object]]::new() + $rightKey = $right.Entries[$index].Key + Add-YamlMergeWork -State $State.WorkState -Node $rightKey ` + -Operation 'equality mapping candidate identity' + $effectiveKey = Get-YamlMergeNode -Node $rightKey + if (-not $rightIndexedIds.Add($effectiveKey.Id)) { + continue + } + $fingerprint = Get-YamlMergeFingerprint -Node $rightKey -State $State ` + -Cache $RightFingerprintCache + Add-YamlMergeWork -State $State.WorkState -Node $rightKey ` + -Operation 'equality mapping bucket visit' + $bucket = $null + if (-not $rightEntries.TryGetValue($fingerprint, [ref] $bucket)) { + $bucket = [pscustomobject]@{ + Candidates = [System.Collections.Generic.List[object]]::new() + } + $rightEntries[$fingerprint] = $bucket } - $rightEntries[$fingerprint].Add([pscustomobject]@{ + Add-YamlMergeWork -State $State.WorkState -Node $rightKey ` + -Operation 'equality mapping candidate visit' + $bucket.Candidates.Add([pscustomobject]@{ Index = $index Entry = $right.Entries[$index] }) } + $matched = [System.Collections.Generic.HashSet[int]]::new() foreach ($leftEntry in $left.Entries) { - $fingerprint = Get-YamlMergeFingerprint -Node $leftEntry.Key -State $State - if (-not $rightEntries.ContainsKey($fingerprint)) { + $fingerprint = Get-YamlMergeFingerprint -Node $leftEntry.Key -State $State ` + -Cache $LeftFingerprintCache + Add-YamlMergeWork -State $State.WorkState -Node $leftEntry.Key ` + -Operation 'equality mapping bucket lookup' + $bucket = $null + if (-not $rightEntries.TryGetValue($fingerprint, [ref] $bucket)) { + $State.Cache[$cacheKey] = $false return $false } $match = $null - foreach ($candidate in $rightEntries[$fingerprint]) { + foreach ($candidate in $bucket.Candidates) { + Add-YamlMergeWork -State $State.WorkState -Node $leftEntry.Key ` + -Operation 'equality mapping candidate comparison' if ($matched.Contains($candidate.Index)) { continue } - if (Test-YamlMergeNodeEqual -Node $leftEntry.Key -OtherNode $candidate.Entry.Key ` - -State $State) { + if (Test-YamlMergeNodeEqual -Node $leftEntry.Key ` + -OtherNode $candidate.Entry.Key -State $State ` + -LeftFingerprintCache $LeftFingerprintCache ` + -RightFingerprintCache $RightFingerprintCache) { $match = $candidate break } } if ($null -eq $match) { + $State.Cache[$cacheKey] = $false return $false } [void] $matched.Add($match.Index) @@ -176,5 +254,6 @@ function Test-YamlMergeNodeEqual { } } + $State.Cache[$cacheKey] = $true return $true } diff --git a/src/functions/public/Merge-Yaml.ps1 b/src/functions/public/Merge-Yaml.ps1 index 0e8e135..a2dd73d 100644 --- a/src/functions/public/Merge-Yaml.ps1 +++ b/src/functions/public/Merge-Yaml.ps1 @@ -47,8 +47,9 @@ function Merge-Yaml { Maximum YAML node nesting depth. The default is 100. .PARAMETER MaxNodes - Maximum nodes per parsed stream and the invocation-wide ceiling for - clone, equality, merge, and result work. The default is 100000. + Maximum nodes per parsed stream and the invocation-wide ceiling applied + independently to clone creation, merge operations, and the result graph. + The default is 100000. .PARAMETER MaxAliases Maximum aliases per parsed stream and in the result. The default is 1000. @@ -206,25 +207,44 @@ function Merge-Yaml { } $fingerprintHasher = [System.Security.Cryptography.SHA256]::Create() + $workState = [pscustomobject]@{ + Count = 0L + MaxNodes = $MaxNodes + } + $mutationState = [pscustomobject]@{ Version = 0L } + $indexDependents = [System.Collections.Generic.Dictionary[int, object]]::new() $equalityState = [pscustomobject]@{ - EqualityCount = 0 MaxNodes = $MaxNodes FingerprintHasher = $fingerprintHasher + WorkState = $workState + MutationState = $mutationState + IndexDependents = $indexDependents + Cache = [System.Collections.Generic.Dictionary[string, bool]]::new( + [System.StringComparer]::Ordinal + ) + InputIndex = 0 } - $workState = [pscustomobject]@{ - MergeCount = 0 - MaxNodes = $MaxNodes - } + $mappingIndexes = [System.Collections.Generic.Dictionary[int, object]]::new() + $sequenceIndexes = [System.Collections.Generic.Dictionary[int, object]]::new() for ($inputIndex = 1; $inputIndex -lt $parsedStreams.Count; $inputIndex++) { $cloneCache = [System.Collections.Generic.Dictionary[int, object]]::new() + $overlayFingerprintCache = ( + [System.Collections.Generic.Dictionary[int, string]]::new() + ) + $equalityState.InputIndex = $inputIndex foreach ($documentIndex in 0..($expectedDocumentCount - 1)) { $context = [pscustomobject]@{ - InputIndex = $inputIndex - DocumentIndex = $documentIndex - CloneCache = $cloneCache - CloneState = $cloneState - EqualityState = $equalityState - WorkState = $workState + InputIndex = $inputIndex + DocumentIndex = $documentIndex + CloneCache = $cloneCache + CloneState = $cloneState + EqualityState = $equalityState + WorkState = $workState + MutationState = $mutationState + MappingIndexes = $mappingIndexes + SequenceIndexes = $sequenceIndexes + IndexDependents = $indexDependents + OverlayFingerprintCache = $overlayFingerprintCache } $resultDocuments[$documentIndex] = Merge-YamlRepresentationNode ` -BaseNode $resultDocuments[$documentIndex] ` @@ -239,6 +259,7 @@ function Merge-Yaml { -MaxAliases $MaxAliases -MaxScalarLength $MaxScalarLength ` -MaxTagLength $MaxTagLength -MaxTotalTagLength $MaxTotalTagLength $merged = ConvertTo-YamlRepresentationText -Documents $resultArray -Indent $Indent + Write-Debug "Merge-Yaml work operations: $($workState.Count)." $PSCmdlet.WriteObject($merged, $false) } catch { $failure = $_ diff --git a/tests/Merge-Yaml.Tests.ps1 b/tests/Merge-Yaml.Tests.ps1 index ef228a4..4d03ef4 100644 --- a/tests/Merge-Yaml.Tests.ps1 +++ b/tests/Merge-Yaml.Tests.ps1 @@ -94,6 +94,38 @@ BeforeAll { } return & $loadedModule $implementation $Yaml } + + function Measure-MergeYamlWork { + <# + .SYNOPSIS + Captures the deterministic merge operation count from the debug stream. + #> + param ( + [Parameter(Mandatory)] + [string[]] $InputObject, + + [Parameter()] + [hashtable] $Parameters = @{} + ) + + $records = @(Merge-Yaml -InputObject $InputObject @Parameters -Debug 5>&1) + $debugText = $records | + ForEach-Object ToString | + Where-Object { $_ -like '*Merge-Yaml work operations:*' } | + Select-Object -Last 1 + if ($null -eq $debugText -or + $debugText -notmatch 'Merge-Yaml work operations: (?\d+)') { + throw 'Merge-Yaml did not report its deterministic work count.' + } + $output = $records | + Where-Object { $_ -isnot [System.Management.Automation.DebugRecord] } | + Select-Object -First 1 + + [pscustomobject]@{ + Output = [string] $output + Count = [long] $Matches.WorkCount + } + } } Describe 'Merge-Yaml' { @@ -243,6 +275,53 @@ tail: base @($result.Values) | Should -Be @('first', 'second') } + It 're-buckets a shared structural key after its target is recursively mutated' ` + -ForEach @( + @{ Action = 'Replace' } + @{ Action = 'Error' } + ) { + $base = "target: &key {x: 1}`n? *key`n: base" + $overlay = "target: {y: 2}`n? {x: 1, y: 2}`n: overlay" + + if ($Action -eq 'Error') { + $failure = Get-MergeYamlFailure { + Merge-Yaml $base, $overlay -ConflictAction Error + } + + $failure.Exception.Data['YamlErrorId'] | + Should -BeExactly 'YamlMergeConflict' + $failure.Exception.Message | Should -Match 'input index 1' + return + } + + $result = Merge-Yaml $base, $overlay | ConvertFrom-Yaml -AsHashtable + + $result.Count | Should -Be 2 + $result['target'].Count | Should -Be 2 + @($result.Values) | Should -Contain 'overlay' + } + + It 're-buckets tagged shared keys across later overlay streams' { + $base = @' +target: &key !item { x: 1 } +? *key +: base +'@ + $firstOverlay = 'target: !item { y: 2 }' + $secondOverlay = @' +? !item { x: 1, y: 2 } +: final +'@ + + $merged = Merge-Yaml $base, $firstOverlay, $secondOverlay + $result = $merged | ConvertFrom-Yaml -AsHashtable + $facts = Get-MergeYamlGraphFact -Yaml $merged + + $result.Count | Should -Be 2 + @($result.Values) | Should -Contain 'final' + @($facts.Tags) | Should -Contain '!item' + } + It 'keeps YAML 1.1 merge syntax as ordinary mapping data' { $base = @' <<: @@ -349,6 +428,48 @@ items: $result['items'][3]['self'] ) | Should -BeTrue } + + It 'invalidates a retained cyclic-item index after shared tagged-node mutation' { + $base = @' +target: &cycle !cycle + self: *cycle + x: 1 +items: [*cycle] +'@ + $firstOverlay = 'items: [added]' + $secondOverlay = @' +target: !cycle { y: 2 } +items: + - &other !cycle + self: *other + x: 1 + y: 2 +'@ + + $merged = Merge-Yaml $base, $firstOverlay, $secondOverlay ` + -SequenceAction Unique + $result = $merged | ConvertFrom-Yaml -AsHashtable + + $result['items'].Count | Should -Be 2 + [object]::ReferenceEquals($result['target'], $result['items'][0]) | + Should -BeTrue + $result['target']['y'] | Should -Be 2 + } + + It 'deduplicates structurally equal cycles with different graph lengths' { + $base = 'items: [&self [*self]]' + $overlay = 'items: [&outer [&inner [*outer]], added]' + + $merged = Merge-Yaml $base, $overlay -SequenceAction Unique + $result = $merged | ConvertFrom-Yaml -AsHashtable + + $result['items'].Count | Should -Be 2 + [object]::ReferenceEquals( + $result['items'][0], + $result['items'][0][0] + ) | Should -BeTrue + $result['items'][1] | Should -Be 'added' + } } Context 'Conflict and null policies' { @@ -570,16 +691,16 @@ node: &cycle Should -Be "$($parseFailure.Exception.Data['YamlErrorId']),Merge-Yaml" } - It 'enforces clone and result node budgets' { + It 'enforces the invocation-wide clone node budget' { $failure = Get-MergeYamlFailure { - Merge-Yaml 'a: 1', 'b: 2' -MaxNodes 3 + Merge-Yaml 'a: 1', '[overlay]' -MaxNodes 3 } $failure.Exception.Data['YamlErrorId'] | Should -BeExactly 'YamlMergeNodeLimitExceeded' } - It 'enforces cumulative equality work budgets' { + It 'charges equality and candidate scans to one invocation work budget' { $base = '[{ a: 1 }, { b: 2 }]' $overlay = '[{ b: 2 }, { a: 1 }]' $failure = Get-MergeYamlFailure { @@ -587,7 +708,128 @@ node: &cycle } $failure.Exception.Data['YamlErrorId'] | - Should -BeExactly 'YamlMergeEqualityLimitExceeded' + Should -BeExactly 'YamlMergeWorkLimitExceeded' + $failure.Exception.Data['YamlMergeWorkCount'] | Should -Be 8 + $failure.Exception.Data['YamlMergeWorkLimit'] | Should -Be 7 + } + + It 'keeps disjoint one-key overlay work linear' -ForEach @( + @{ Size = 200 } + @{ Size = 400 } + @{ Size = 800 } + ) { + $streams = [System.Collections.Generic.List[string]]::new() + $streams.Add('base: true') + foreach ($index in 1..$Size) { + $streams.Add("key${index}: ${index}") + } + $limit = 12 * $Size + 50 + + $measurement = Measure-MergeYamlWork -InputObject $streams.ToArray() ` + -Parameters @{ MaxNodes = $limit } + + $measurement.Count | Should -BeLessOrEqual (10 * $Size + 20) + $measurement.Output | Should -Not -BeNullOrEmpty + } + + It 'keeps unique sequence append work linear' -ForEach @( + @{ Size = 200 } + @{ Size = 400 } + @{ Size = 800 } + ) { + $streams = [System.Collections.Generic.List[string]]::new() + $streams.Add('items: []') + foreach ($index in 1..$Size) { + $streams.Add("items: [value${index}]") + } + $limit = 31 * $Size + 100 + + $measurement = Measure-MergeYamlWork -InputObject $streams.ToArray() ` + -Parameters @{ MaxNodes = $limit; SequenceAction = 'Unique' } + + $measurement.Count | Should -BeLessOrEqual (30 * $Size + 20) + $measurement.Output | Should -Not -BeNullOrEmpty + } + + It 'reuses fingerprints for shared complex keys throughout equality' -ForEach @( + @{ Size = 20 } + @{ Size = 40 } + @{ Size = 80 } + ) { + $values = (1..$Size | ForEach-Object { "value$_" }) -join ', ' + $yaml = [System.Collections.Generic.List[string]]::new() + $yaml.Add("shared: &key [$values]") + $yaml.Add('maps:') + foreach ($index in 1..$Size) { + $yaml.Add(' - ? *key') + $yaml.Add(" : value${index}") + } + $text = $yaml -join "`n" + + $measurement = Measure-MergeYamlWork -InputObject @($text, $text) ` + -Parameters @{ MaxNodes = 100000; ConflictAction = 'Error' } + + $measurement.Count | Should -BeLessOrEqual (20 * $Size + 50) + $measurement.Output | Should -Not -BeNullOrEmpty + } + + It 'deduplicates cyclic alias candidates before fingerprint traversal' -ForEach @( + @{ Size = 20 } + @{ Size = 40 } + @{ Size = 80 } + ) { + $base = [System.Collections.Generic.List[string]]::new() + $base.Add('items:') + $base.Add(' - &shared') + $base.Add(' self: *shared') + foreach ($index in 1..$Size) { + $base.Add(" field${index}: ${index}") + } + foreach ($index in 2..$Size) { + $base.Add(' - *shared') + } + $overlay = [System.Collections.Generic.List[string]]::new() + foreach ($line in $base) { + $overlay.Add($line) + } + $overlay.Add(' - added') + + $measurement = Measure-MergeYamlWork -InputObject @( + $base -join "`n" + $overlay -join "`n" + ) -Parameters @{ MaxNodes = 100000; SequenceAction = 'Unique' } + + $measurement.Count | Should -BeLessOrEqual (21 * $Size + 100) + $measurement.Output | Should -Not -BeNullOrEmpty + } + + It 'deduplicates hundreds of aliases before comparing one large mapping' { + $aliasCount = 300 + $entryCount = 200 + $base = [System.Collections.Generic.List[string]]::new() + $base.Add('shared: &shared') + foreach ($index in 1..$entryCount) { + $base.Add(" item${index}: ${index}") + } + foreach ($index in 1..$aliasCount) { + $base.Add("alias${index}: *shared") + } + $overlay = [System.Collections.Generic.List[string]]::new() + $overlay.Add('shared: &shared') + $overlay.Add(' added: true') + foreach ($index in 1..$aliasCount) { + $overlay.Add("alias${index}: *shared") + } + + $measurement = Measure-MergeYamlWork -InputObject @( + $base -join "`n" + $overlay -join "`n" + ) -Parameters @{ MaxNodes = 12000 } + $result = $measurement.Output | ConvertFrom-Yaml -AsHashtable + + $measurement.Count | Should -BeLessOrEqual 7000 + [object]::ReferenceEquals($result['shared'], $result['alias300']) | + Should -BeTrue } It 'enforces resulting alias and tag budgets' -ForEach @( From f4b418fee596e3889402a7e2f9028702f0f74ffd Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sat, 25 Jul 2026 08:59:01 +0200 Subject: [PATCH 8/8] Clarify YAML merge operation budgets Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 21ac612..0c8dc33 100644 --- a/README.md +++ b/README.md @@ -191,8 +191,8 @@ $mergedYaml = Merge-Yaml -InputObject @($baseYaml, $overlayYaml) Compatible mappings merge recursively by structural YAML key equality. Base key order remains stable, replacing a value retains its position, and new overlay keys append in overlay order. Complex and tagged keys are supported. Structural -fingerprints select comparison candidates only; graph-aware equality makes the -final key decision. +fingerprints select comparison candidates only; mutation-aware indexes are +retained across overlays, and graph-aware equality makes the final key decision. Compatible sequences use `-SequenceAction Replace`, `Append`, or `Unique`. Unequal scalars, collection kinds, and incompatible effective tags use @@ -212,9 +212,10 @@ one deterministic string with LF line endings, explicit document starts, and no final newline. The parser safety parameters and defaults match `Format-Yaml`. `-MaxNodes` -limits each parsed stream and invocation-wide cloning, equality, merge work, and -the resulting stream graph. Alias and expanded-tag budgets are also enforced on -the result. +limits each parsed stream and applies independently to invocation-wide clone +creation, charged merge operations, and the resulting stream graph. Index, +fingerprint, candidate, alias-traversal, and equality work all consume the merge +operation budget. Alias and expanded-tag budgets are also enforced on the result. ## Export YAML files