From a580b6f8288f56023c35a49144df088de8ac58a6 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 23 Jul 2026 22:53:51 +0200 Subject: [PATCH 01/24] Harden YAML lexical boundaries Use YAML s-white and ordinal token checks, consume legal BOMs, preserve flow scalar folding, and disambiguate anchor and alias colons without regressing the official corpus. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../private/ConvertFrom-YamlByteOrderMark.ps1 | 55 +++++++++ .../private/Find-YamlCommentStart.ps1 | 22 ++++ .../private/Find-YamlMappingColon.ps1 | 30 +++-- .../private/Get-YamlContentWithoutComment.ps1 | 8 +- .../private/New-YamlReaderContext.ps1 | 3 +- src/functions/private/Read-YamlBlockKey.ps1 | 4 +- .../private/Read-YamlBlockMapping.ps1 | 32 ++--- src/functions/private/Read-YamlBlockNode.ps1 | 23 ++-- .../private/Read-YamlBlockSequence.ps1 | 9 +- .../private/Read-YamlDirectiveBlock.ps1 | 2 +- src/functions/private/Read-YamlFlowNode.ps1 | 114 +++++++++++------- .../private/Read-YamlNodeProperty.ps1 | 16 ++- .../private/Read-YamlPlainScalar.ps1 | 12 +- src/functions/private/Read-YamlStreamCore.ps1 | 21 ++-- src/functions/private/Resolve-YamlScalar.ps1 | 4 +- src/functions/private/Skip-YamlFlowTrivia.ps1 | 5 +- src/functions/private/Test-YamlIndicator.ps1 | 14 ++- .../Test-YamlMappingValueIndicator.ps1 | 31 +++++ src/functions/private/Test-YamlTagUriText.ps1 | 3 +- src/functions/private/Test-YamlWhiteSpace.ps1 | 14 +++ tests/ConvertFrom-Yaml.Tests.ps1 | 40 ++++++ 21 files changed, 346 insertions(+), 116 deletions(-) create mode 100644 src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 create mode 100644 src/functions/private/Find-YamlCommentStart.ps1 create mode 100644 src/functions/private/Test-YamlMappingValueIndicator.ps1 create mode 100644 src/functions/private/Test-YamlWhiteSpace.ps1 diff --git a/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 b/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 new file mode 100644 index 0000000..ae12d96 --- /dev/null +++ b/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 @@ -0,0 +1,55 @@ +function ConvertFrom-YamlByteOrderMark { + <# + .SYNOPSIS + Consumes byte order marks at YAML stream and explicit document boundaries. + #> + [CmdletBinding()] + [OutputType([string])] + param ( + [Parameter(Mandatory)] + [AllowEmptyString()] + [string] $Text + ) + + if ($Text.IndexOf([char] 0xFEFF) -lt 0) { + return $Text + } + + $builder = [System.Text.StringBuilder]::new($Text.Length) + for ($index = 0; $index -lt $Text.Length; $index++) { + if ($Text[$index] -cne [char] 0xFEFF) { + [void] $builder.Append($Text[$index]) + continue + } + + $atStreamStart = $index -eq 0 + $atLineStart = $index -gt 0 -and $Text[$index - 1] -ceq "`n" + $beforeDirective = $index + 1 -lt $Text.Length -and $Text[$index + 1] -ceq '%' + $beforeDocumentStart = $false + if ($index + 3 -lt $Text.Length -and + $Text.Substring($index + 1, 3).Equals( + '---', + [System.StringComparison]::Ordinal + )) { + $afterMarker = $index + 4 + $beforeDocumentStart = $afterMarker -ge $Text.Length -or + (Test-YamlWhiteSpace -Character $Text[$afterMarker]) -or + $Text[$afterMarker] -ceq "`n" + } + + if ($atStreamStart -or ($atLineStart -and ($beforeDirective -or $beforeDocumentStart))) { + continue + } + + $before = $Text.Substring(0, $index) + $line = ([regex]::Matches($before, "`n")).Count + $lastBreak = $before.LastIndexOf("`n", [System.StringComparison]::Ordinal) + $column = if ($lastBreak -lt 0) { $before.Length } else { $before.Length - $lastBreak - 1 } + $mark = New-YamlMark -Index $index -Line $line -Column $column + throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidByteOrderMark' -Message ( + 'A YAML byte order mark is only allowed at the start of the stream or an explicit document.' + )) + } + + return $builder.ToString() +} diff --git a/src/functions/private/Find-YamlCommentStart.ps1 b/src/functions/private/Find-YamlCommentStart.ps1 new file mode 100644 index 0000000..a0d08f7 --- /dev/null +++ b/src/functions/private/Find-YamlCommentStart.ps1 @@ -0,0 +1,22 @@ +function Find-YamlCommentStart { + <# + .SYNOPSIS + Finds a comment indicator separated by YAML s-white. + #> + [CmdletBinding()] + [OutputType([int])] + param ( + [Parameter(Mandatory)] + [AllowEmptyString()] + [string] $Text + ) + + for ($index = 0; $index -lt $Text.Length; $index++) { + if ($Text[$index] -ceq '#' -and ( + $index -eq 0 -or (Test-YamlWhiteSpace -Character $Text[$index - 1]) + )) { + return $index + } + } + return -1 +} diff --git a/src/functions/private/Find-YamlMappingColon.ps1 b/src/functions/private/Find-YamlMappingColon.ps1 index bb6ba59..9f2fc66 100644 --- a/src/functions/private/Find-YamlMappingColon.ps1 +++ b/src/functions/private/Find-YamlMappingColon.ps1 @@ -8,7 +8,9 @@ function Find-YamlMappingColon { param ( [Parameter(Mandatory)] [AllowEmptyString()] - [string] $Text + [string] $Text, + + [switch] $AllowAnchorFallback ) if ($Text.IndexOf(':') -lt 0) { @@ -19,6 +21,7 @@ function Find-YamlMappingColon { $singleQuoted = $false $doubleQuoted = $false $atNodeStart = $true + $anchorColonCandidate = -1 for ($index = 0; $index -lt $Text.Length; $index++) { $character = $Text[$index] if ($doubleQuoted) { @@ -39,12 +42,17 @@ function Find-YamlMappingColon { } continue } - if ($atNodeStart -and [char]::IsWhiteSpace($character)) { + if ($atNodeStart -and (Test-YamlWhiteSpace -Character $character)) { continue } if ($atNodeStart -and $character -eq '&') { $index++ - while ($index -lt $Text.Length -and -not [char]::IsWhiteSpace($Text[$index])) { + while ($index -lt $Text.Length -and + -not (Test-YamlWhiteSpace -Character $Text[$index]) -and + $Text[$index] -notin @(',', '[', ']', '{', '}')) { + if (Test-YamlMappingValueIndicator -Text $Text -Index $index) { + $anchorColonCandidate = $index + } $index++ } $index-- @@ -52,7 +60,9 @@ function Find-YamlMappingColon { } if ($atNodeStart -and $character -eq '*') { $index++ - while ($index -lt $Text.Length -and -not [char]::IsWhiteSpace($Text[$index])) { + while ($index -lt $Text.Length -and + -not (Test-YamlWhiteSpace -Character $Text[$index]) -and + $Text[$index] -notin @(',', '[', ']', '{', '}')) { $index++ } $index-- @@ -67,7 +77,7 @@ function Find-YamlMappingColon { } else { $index++ while ($index -lt $Text.Length -and - -not [char]::IsWhiteSpace($Text[$index]) -and + -not (Test-YamlWhiteSpace -Character $Text[$index]) -and $Text[$index] -notin @('[', ']', '{', '}', ',')) { $index++ } @@ -94,18 +104,24 @@ function Find-YamlMappingColon { ':' { if ($flowDepth -eq 0) { $nextIsSeparator = $index + 1 -ge $Text.Length - $nextIsSeparator = $nextIsSeparator -or [char]::IsWhiteSpace($Text[$index + 1]) + $nextIsSeparator = $nextIsSeparator -or + (Test-YamlWhiteSpace -Character $Text[$index + 1]) if ($nextIsSeparator) { return $index } } } '#' { - if ($flowDepth -eq 0 -and ($index -eq 0 -or [char]::IsWhiteSpace($Text[$index - 1]))) { + if ($flowDepth -eq 0 -and ( + $index -eq 0 -or (Test-YamlWhiteSpace -Character $Text[$index - 1]) + )) { return -1 } } } } + if ($AllowAnchorFallback) { + return $anchorColonCandidate + } return -1 } diff --git a/src/functions/private/Get-YamlContentWithoutComment.ps1 b/src/functions/private/Get-YamlContentWithoutComment.ps1 index 61c44eb..9f39bb4 100644 --- a/src/functions/private/Get-YamlContentWithoutComment.ps1 +++ b/src/functions/private/Get-YamlContentWithoutComment.ps1 @@ -11,9 +11,9 @@ function Get-YamlContentWithoutComment { [string] $Text ) - $comment = [regex]::Match($Text, '(?', "'", '"', '%', '@', '`') -or ($first -in @('-', '?', ':') -and ( - $rest.Length -gt 1 -and [char]::IsWhiteSpace($rest[1]) + $rest.Length -gt 1 -and (Test-YamlWhiteSpace -Character $rest[1]) ))) { throw (New-YamlException -Start $start -End $start -ErrorId 'YamlInvalidPlainScalar' -Message ( 'The first character is not allowed in a YAML plain scalar.' @@ -82,7 +82,7 @@ function Read-YamlBlockKey { $node = New-YamlSyntaxNode -Context $Context -Kind Scalar -Depth $Depth -Start $start -End $end Set-YamlParsedNodeProperty -Node $node -Tag $properties.Tag ` -HasUnknownTag $properties.HasUnknownTag -Anchor $properties.Anchor -Context $Context - $node.Value = $rest.TrimEnd() + $node.Value = $rest.TrimEnd(' ', "`t") $node.Style = 'Plain' $node.IsPlainImplicit = [string]::IsNullOrEmpty($properties.Tag) -and -not $properties.HasUnknownTag diff --git a/src/functions/private/Read-YamlBlockMapping.ps1 b/src/functions/private/Read-YamlBlockMapping.ps1 index bc64bc0..8e8177b 100644 --- a/src/functions/private/Read-YamlBlockMapping.ps1 +++ b/src/functions/private/Read-YamlBlockMapping.ps1 @@ -77,10 +77,10 @@ function Read-YamlBlockMapping { $isExplicit = Test-YamlIndicator -Text $content -Indicator '?' if ($isExplicit) { $afterQuestion = $content.Substring(1) - $leading = $afterQuestion.Length - $afterQuestion.TrimStart().Length - $keyText = $afterQuestion.TrimStart() + $leading = $afterQuestion.Length - $afterQuestion.TrimStart(' ', "`t").Length + $keyText = $afterQuestion.TrimStart(' ', "`t") $keyColumn = $contentColumn + 1 + $leading - if ([string]::IsNullOrWhiteSpace((Get-YamlContentWithoutComment -Text $keyText))) { + if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $keyText))) { $Context.LineIndex++ Skip-YamlBlockTrivia -Context $Context if ($Context.LineIndex -ge $Context.Lines.Count) { @@ -104,9 +104,12 @@ function Read-YamlBlockMapping { } } } elseif (Test-YamlIndicator -Text $keyText -Indicator '-') { - $firstItem = $keyText.Substring(1).TrimStart() + $firstItem = $keyText.Substring(1).TrimStart(' ', "`t") $dashColumn = $keyColumn - $itemColumn = $dashColumn + 1 + ($keyText.Substring(1).Length - $keyText.Substring(1).TrimStart().Length) + $itemColumn = $dashColumn + 1 + ( + $keyText.Substring(1).Length - + $keyText.Substring(1).TrimStart(' ', "`t").Length + ) $key = Read-YamlBlockSequence -Context $Context -Indent $dashColumn -Depth ($Depth + 1) ` -FirstItemText $firstItem -FirstItemColumn $itemColumn } else { @@ -130,10 +133,10 @@ function Read-YamlBlockMapping { if ($valueIndent -eq $Indent -and (Test-YamlIndicator -Text $valueContent -Indicator ':')) { $valueText = $valueContent.Substring(1) - $leading = $valueText.Length - $valueText.TrimStart().Length - $valueText = $valueText.TrimStart() + $leading = $valueText.Length - $valueText.TrimStart(' ', "`t").Length + $valueText = $valueText.TrimStart(' ', "`t") $valueColumn = $Indent + 1 + $leading - if ([string]::IsNullOrWhiteSpace((Get-YamlContentWithoutComment -Text $valueText))) { + if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $valueText))) { $valueLineNumber = $Context.LineIndex $Context.LineIndex++ Skip-YamlBlockTrivia -Context $Context @@ -171,7 +174,7 @@ function Read-YamlBlockMapping { continue } - $colon = Find-YamlMappingColon -Text $content + $colon = Find-YamlMappingColon -Text $content -AllowAnchorFallback if ($colon -lt 0) { $mark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $contentColumn) ` -Line $lineNumber -Column $contentColumn @@ -192,17 +195,17 @@ function Read-YamlBlockMapping { -Line $lineNumber -Column $contentColumn $key = New-YamlEmptyScalar -Context $Context -Depth ($Depth + 1) -Mark $mark } else { - $keyText = $content.Substring(0, $colon).TrimEnd() + $keyText = $content.Substring(0, $colon).TrimEnd(' ', "`t") $key = Read-YamlBlockKey -Context $Context -Text $keyText -Line $lineNumber ` -Column $contentColumn -Depth ($Depth + 1) } $valueStart = $colon + 1 $valueSource = $content.Substring($valueStart) - $leading = $valueSource.Length - $valueSource.TrimStart().Length - $valueText = $valueSource.TrimStart() + $leading = $valueSource.Length - $valueSource.TrimStart(' ', "`t").Length + $valueText = $valueSource.TrimStart(' ', "`t") $valueColumn = $contentColumn + $valueStart + $leading - if ([string]::IsNullOrWhiteSpace((Get-YamlContentWithoutComment -Text $valueText))) { + if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $valueText))) { $Context.LineIndex = $lineNumber + 1 Skip-YamlBlockTrivia -Context $Context if ($Context.LineIndex -lt $Context.Lines.Count) { @@ -242,7 +245,8 @@ function Read-YamlBlockMapping { } $Context.LineIndex = $lineNumber $value = Read-YamlBlockNode -Context $Context -ParentIndent $Indent -Depth ($Depth + 1) ` - -Segment $valueText -SegmentColumn $valueColumn -AllowIndentlessSequence + -Segment $valueText -SegmentColumn $valueColumn -AllowIndentlessSequence ` + -DisallowCompactMapping } $node.Entries.Add([pscustomobject]@{ Key = $key; Value = $value }) } diff --git a/src/functions/private/Read-YamlBlockNode.ps1 b/src/functions/private/Read-YamlBlockNode.ps1 index 972c605..cdc39a4 100644 --- a/src/functions/private/Read-YamlBlockNode.ps1 +++ b/src/functions/private/Read-YamlBlockNode.ps1 @@ -29,7 +29,9 @@ function Read-YamlBlockNode { [AllowEmptyString()] [string] $PendingAnchor = '', - [switch] $AllowIndentlessSequence + [switch] $AllowIndentlessSequence, + + [switch] $DisallowCompactMapping ) $hasSegment = $PSBoundParameters.ContainsKey('Segment') @@ -67,8 +69,9 @@ function Read-YamlBlockNode { if ( (-not [string]::IsNullOrEmpty($PendingTag) -or $PendingUnknownTag -or -not [string]::IsNullOrEmpty($PendingAnchor)) -and + -not $DisallowCompactMapping -and -not (Test-YamlIndicator -Text $Segment -Indicator '-') -and - ((Find-YamlMappingColon -Text $Segment) -ge 0 -or + ((Find-YamlMappingColon -Text $Segment -AllowAnchorFallback) -ge 0 -or (Test-YamlIndicator -Text $Segment -Indicator '?')) ) { return Read-YamlBlockMapping -Context $Context -Indent $SegmentColumn -Depth $Depth ` @@ -78,8 +81,9 @@ function Read-YamlBlockNode { if ( [string]::IsNullOrEmpty($PendingTag) -and -not $PendingUnknownTag -and [string]::IsNullOrEmpty($PendingAnchor) -and + -not $DisallowCompactMapping -and -not (Test-YamlIndicator -Text $Segment -Indicator '-') -and - ((Find-YamlMappingColon -Text $Segment) -ge 0 -or + ((Find-YamlMappingColon -Text $Segment -AllowAnchorFallback) -ge 0 -or (Test-YamlIndicator -Text $Segment -Indicator '?')) ) { return Read-YamlBlockMapping -Context $Context -Indent $SegmentColumn -Depth $Depth ` @@ -111,7 +115,7 @@ function Read-YamlBlockNode { $rest = Get-YamlContentWithoutComment -Text $restSource $contentColumn = $SegmentColumn + $properties.Consumed - if ([string]::IsNullOrWhiteSpace($rest)) { + if ([string]::IsNullOrEmpty($rest)) { $Context.LineIndex++ Skip-YamlBlockTrivia -Context $Context if ($Context.LineIndex -ge $Context.Lines.Count -or @@ -154,7 +158,7 @@ function Read-YamlBlockNode { } $firstSource = $rest.Substring(1) if ($firstSource.IndexOf("`t", [System.StringComparison]::Ordinal) -ge 0 -and - $firstSource.Trim() -eq '-') { + $firstSource.Trim(' ', "`t") -ceq '-') { $mark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $contentColumn + 1) ` -Line $lineNumber -Column ($contentColumn + 1) throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidIndentation' -Message ( @@ -162,15 +166,16 @@ function Read-YamlBlockNode { )) } $first = $firstSource - $leading = $first.Length - $first.TrimStart().Length - $first = $first.TrimStart() + $leading = $first.Length - $first.TrimStart(' ', "`t").Length + $first = $first.TrimStart(' ', "`t") return Read-YamlBlockSequence -Context $Context -Indent $contentColumn -Depth $Depth -Tag $tag ` -HasUnknownTag $unknownTag -Anchor $anchor -FirstItemText $first ` -FirstItemColumn ($contentColumn + 1 + $leading) } - if ((Find-YamlMappingColon -Text $rest) -ge 0 -or - (Test-YamlIndicator -Text $rest -Indicator '?')) { + if (-not $DisallowCompactMapping -and ( + (Find-YamlMappingColon -Text $rest -AllowAnchorFallback) -ge 0 -or + (Test-YamlIndicator -Text $rest -Indicator '?'))) { return Read-YamlBlockMapping -Context $Context -Indent $contentColumn -Depth $Depth -Tag $tag ` -HasUnknownTag $unknownTag -Anchor $anchor -FirstText $rest -FirstColumn $contentColumn } diff --git a/src/functions/private/Read-YamlBlockSequence.ps1 b/src/functions/private/Read-YamlBlockSequence.ps1 index 5ca2bba..3c272b4 100644 --- a/src/functions/private/Read-YamlBlockSequence.ps1 +++ b/src/functions/private/Read-YamlBlockSequence.ps1 @@ -62,11 +62,14 @@ function Read-YamlBlockSequence { if (-not (Test-YamlIndicator -Text $content -Indicator '-')) { break } - $rest = $content.Substring(1).TrimStart() - $itemColumn = $Indent + 1 + ($content.Substring(1).Length - $content.Substring(1).TrimStart().Length) + $rest = $content.Substring(1).TrimStart(' ', "`t") + $itemColumn = $Indent + 1 + ( + $content.Substring(1).Length - + $content.Substring(1).TrimStart(' ', "`t").Length + ) } - if ([string]::IsNullOrWhiteSpace((Get-YamlContentWithoutComment -Text $rest))) { + if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $rest))) { $line = $Context.LineIndex $Context.LineIndex++ Skip-YamlBlockTrivia -Context $Context diff --git a/src/functions/private/Read-YamlDirectiveBlock.ps1 b/src/functions/private/Read-YamlDirectiveBlock.ps1 index 7b7b97d..37853d1 100644 --- a/src/functions/private/Read-YamlDirectiveBlock.ps1 +++ b/src/functions/private/Read-YamlDirectiveBlock.ps1 @@ -71,7 +71,7 @@ function Read-YamlDirectiveBlock { } $tagHandles[$handle] = $prefix } elseif ($directive -cnotmatch ( - '^%[A-Za-z0-9]+(?:[ \t]+[^#\s][^#]*?)?(?:[ \t]+#.*)?$' + '^%[A-Za-z0-9]+(?:[ \t]+[^# \t][^#]*?)?(?:[ \t]+#.*)?$' )) { throw (New-YamlException -Start $mark -End $mark ` -ErrorId 'YamlInvalidDirective' -Message ( diff --git a/src/functions/private/Read-YamlFlowNode.ps1 b/src/functions/private/Read-YamlFlowNode.ps1 index 0015d34..7a74b76 100644 --- a/src/functions/private/Read-YamlFlowNode.ps1 +++ b/src/functions/private/Read-YamlFlowNode.ps1 @@ -22,7 +22,9 @@ function Read-YamlFlowNode { [bool] $PendingUnknownTag = $false, [AllowEmptyString()] - [string] $PendingAnchor = '' + [string] $PendingAnchor = '', + + [switch] $InFlowCollection ) Skip-YamlFlowTrivia -Cursor $Cursor -Context $Context @@ -52,7 +54,8 @@ function Read-YamlFlowNode { Move-YamlCursor -Cursor $Cursor -Context $Context } else { while ($Cursor.Index -lt $Context.Text.Length -and - -not [char]::IsWhiteSpace($Context.Text[$Cursor.Index]) -and + -not (Test-YamlWhiteSpace -Character $Context.Text[$Cursor.Index]) -and + $Context.Text[$Cursor.Index] -cne "`n" -and $Context.Text[$Cursor.Index] -notin @(',', '[', ']', '{', '}')) { Move-YamlCursor -Cursor $Cursor -Context $Context } @@ -79,8 +82,11 @@ function Read-YamlFlowNode { Move-YamlCursor -Cursor $Cursor -Context $Context $anchorStart = $Cursor.Index while ($Cursor.Index -lt $Context.Text.Length -and - -not [char]::IsWhiteSpace($Context.Text[$Cursor.Index]) -and - $Context.Text[$Cursor.Index] -notin @(',', '[', ']', '{', '}')) { + -not (Test-YamlWhiteSpace -Character $Context.Text[$Cursor.Index]) -and + $Context.Text[$Cursor.Index] -cne "`n" -and + $Context.Text[$Cursor.Index] -notin @(',', '[', ']', '{', '}') -and + -not ($InFlowCollection -and + (Test-YamlMappingValueIndicator -Text $Context.Text -Index $Cursor.Index -Flow))) { Move-YamlCursor -Cursor $Cursor -Context $Context } $anchor = $Context.Text.Substring($anchorStart, $Cursor.Index - $anchorStart) @@ -123,8 +129,11 @@ function Read-YamlFlowNode { Move-YamlCursor -Cursor $Cursor -Context $Context $aliasStart = $Cursor.Index while ($Cursor.Index -lt $Context.Text.Length -and - -not [char]::IsWhiteSpace($Context.Text[$Cursor.Index]) -and - $Context.Text[$Cursor.Index] -notin @(',', '[', ']', '{', '}')) { + -not (Test-YamlWhiteSpace -Character $Context.Text[$Cursor.Index]) -and + $Context.Text[$Cursor.Index] -cne "`n" -and + $Context.Text[$Cursor.Index] -notin @(',', '[', ']', '{', '}') -and + -not ($InFlowCollection -and + (Test-YamlMappingValueIndicator -Text $Context.Text -Index $Cursor.Index -Flow))) { Move-YamlCursor -Cursor $Cursor -Context $Context } $alias = $Context.Text.Substring($aliasStart, $Cursor.Index - $aliasStart) @@ -159,24 +168,22 @@ function Read-YamlFlowNode { $explicitPair = $false if ($Context.Text[$Cursor.Index] -eq '?' -and ($Cursor.Index + 1 -ge $Context.Text.Length -or - [char]::IsWhiteSpace($Context.Text[$Cursor.Index + 1]))) { + (Test-YamlWhiteSpace -Character $Context.Text[$Cursor.Index + 1]) -or + $Context.Text[$Cursor.Index + 1] -ceq "`n")) { $explicitPair = $true Move-YamlCursor -Cursor $Cursor -Context $Context Skip-YamlFlowTrivia -Cursor $Cursor -Context $Context } if ($Cursor.Index -lt $Context.Text.Length -and - $Context.Text[$Cursor.Index] -eq ':' -and ( - $Cursor.Index + 1 -ge $Context.Text.Length -or - [char]::IsWhiteSpace($Context.Text[$Cursor.Index + 1]) -or - $Context.Text[$Cursor.Index + 1] -in @(',', ']', '}') - )) { + (Test-YamlMappingValueIndicator -Text $Context.Text -Index $Cursor.Index -Flow)) { $itemMark = New-YamlMark -Index $Cursor.Index -Line $Cursor.Line -Column $Cursor.Column $item = New-YamlSyntaxNode -Context $Context -Kind Scalar -Depth ($Depth + 1) ` -Start $itemMark -End $itemMark $item.Value = '' $item.IsPlainImplicit = $true } else { - $item = Read-YamlFlowNode -Cursor $Cursor -Context $Context -Depth ($Depth + 1) + $item = Read-YamlFlowNode -Cursor $Cursor -Context $Context -Depth ($Depth + 1) ` + -InFlowCollection } $itemStartLine = $item.Start.Line Skip-YamlFlowTrivia -Cursor $Cursor -Context $Context @@ -202,7 +209,8 @@ function Read-YamlFlowNode { $value.Value = '' $value.IsPlainImplicit = $true } else { - $value = Read-YamlFlowNode -Cursor $Cursor -Context $Context -Depth ($Depth + 2) + $value = Read-YamlFlowNode -Cursor $Cursor -Context $Context -Depth ($Depth + 2) ` + -InFlowCollection } $pair.Entries.Add([pscustomobject]@{ Key = $item; Value = $value }) $item = $pair @@ -236,7 +244,8 @@ function Read-YamlFlowNode { $explicit = $false if ($Context.Text[$Cursor.Index] -eq '?' -and ($Cursor.Index + 1 -ge $Context.Text.Length -or - [char]::IsWhiteSpace($Context.Text[$Cursor.Index + 1]))) { + (Test-YamlWhiteSpace -Character $Context.Text[$Cursor.Index + 1]) -or + $Context.Text[$Cursor.Index + 1] -ceq "`n")) { $explicit = $true Move-YamlCursor -Cursor $Cursor -Context $Context Skip-YamlFlowTrivia -Cursor $Cursor -Context $Context @@ -255,7 +264,8 @@ function Read-YamlFlowNode { $key.Value = '' $key.IsPlainImplicit = $true } else { - $key = Read-YamlFlowNode -Cursor $Cursor -Context $Context -Depth ($Depth + 1) + $key = Read-YamlFlowNode -Cursor $Cursor -Context $Context -Depth ($Depth + 1) ` + -InFlowCollection } } Skip-YamlFlowTrivia -Cursor $Cursor -Context $Context @@ -298,7 +308,8 @@ function Read-YamlFlowNode { $value.Value = '' $value.IsPlainImplicit = $true } else { - $value = Read-YamlFlowNode -Cursor $Cursor -Context $Context -Depth ($Depth + 1) + $value = Read-YamlFlowNode -Cursor $Cursor -Context $Context -Depth ($Depth + 1) ` + -InFlowCollection } } $node.Entries.Add([pscustomobject]@{ Key = $key; Value = $value }) @@ -462,9 +473,10 @@ function Read-YamlFlowNode { } if ($Cursor.Column -eq 0 -and $Cursor.Index + 3 -le $Context.Text.Length) { $marker = $Context.Text.Substring($Cursor.Index, 3) - if ($marker -in @('---', '...') -and + if ($marker -cin @('---', '...') -and ($Cursor.Index + 3 -eq $Context.Text.Length -or - [char]::IsWhiteSpace($Context.Text[$Cursor.Index + 3]))) { + (Test-YamlWhiteSpace -Character $Context.Text[$Cursor.Index + 3]) -or + $Context.Text[$Cursor.Index + 3] -ceq "`n")) { $mark = New-YamlMark -Index $Cursor.Index -Line $Cursor.Line -Column 0 throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidQuotedScalar' -Message ( 'A document marker cannot occur inside a multiline quoted scalar.' @@ -513,13 +525,15 @@ function Read-YamlFlowNode { } $builder = [System.Text.StringBuilder]::new() - $pendingSpace = $false + $pendingWhiteSpace = [System.Text.StringBuilder]::new() + $pendingBreaks = 0 $lastContentLine = $Cursor.Line $firstPlainCharacter = $Context.Text[$Cursor.Index] if ($firstPlainCharacter -in @(',', '[', ']', '{', '}', '#', '&', '*', '!', '|', '>', "'", '"', '%', '@', '`') -or ($firstPlainCharacter -in @('-', '?', ':') -and ( $Cursor.Index + 1 -ge $Context.Text.Length -or - [char]::IsWhiteSpace($Context.Text[$Cursor.Index + 1]) -or + (Test-YamlWhiteSpace -Character $Context.Text[$Cursor.Index + 1]) -or + $Context.Text[$Cursor.Index + 1] -ceq "`n" -or $Context.Text[$Cursor.Index + 1] -in @(',', '[', ']', '{', '}') ))) { throw (New-YamlException -Start $start -End $start -ErrorId 'YamlInvalidPlainScalar' -Message ( @@ -531,46 +545,58 @@ function Read-YamlFlowNode { if ($character -in @(',', '[', ']', '{', '}')) { break } - if ($character -eq ':' -and ( - $Cursor.Index + 1 -ge $Context.Text.Length -or - [char]::IsWhiteSpace($Context.Text[$Cursor.Index + 1]) -or - $Context.Text[$Cursor.Index + 1] -in @(',', ']', '}') - )) { + if ($InFlowCollection -and + (Test-YamlMappingValueIndicator -Text $Context.Text -Index $Cursor.Index -Flow)) { break } if ($character -eq '#' -and - ($Cursor.Index -eq 0 -or [char]::IsWhiteSpace($Context.Text[$Cursor.Index - 1]))) { + ($Cursor.Index -eq 0 -or + (Test-YamlWhiteSpace -Character $Context.Text[$Cursor.Index - 1]) -or + $Context.Text[$Cursor.Index - 1] -ceq "`n")) { break } - if ([char]::IsWhiteSpace($character)) { - $pendingSpace = $true + if (Test-YamlWhiteSpace -Character $character) { + [void] $pendingWhiteSpace.Append($character) Move-YamlCursor -Cursor $Cursor -Context $Context - if ($character -eq "`n") { + continue + } + if ($character -eq "`n") { + $pendingWhiteSpace.Clear() | Out-Null + $pendingBreaks = 0 + while ($Cursor.Index -lt $Context.Text.Length -and + $Context.Text[$Cursor.Index] -ceq "`n") { + $pendingBreaks++ + Move-YamlCursor -Cursor $Cursor -Context $Context $indentSpaces = 0 while ($Cursor.Index -lt $Context.Text.Length -and - $Context.Text[$Cursor.Index] -eq ' ') { + $Context.Text[$Cursor.Index] -ceq ' ') { $indentSpaces++ Move-YamlCursor -Cursor $Cursor -Context $Context } while ($Cursor.Index -lt $Context.Text.Length -and - $Context.Text[$Cursor.Index] -eq "`t") { + $Context.Text[$Cursor.Index] -ceq "`t") { Move-YamlCursor -Cursor $Cursor -Context $Context } - if ($Cursor.Index -lt $Context.Text.Length -and - $Context.Text[$Cursor.Index] -notin @("`n", '#', ']', '}') -and - $indentSpaces -le $Cursor.ParentIndent) { - $mark = New-YamlMark -Index $Cursor.Index -Line $Cursor.Line -Column $Cursor.Column - throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidFlowIndentation' -Message ( - 'A multiline flow scalar must be indented beyond its surrounding block context.' - )) - } + } + if ($Cursor.Index -lt $Context.Text.Length -and + $Context.Text[$Cursor.Index] -notin @("`n", '#', ']', '}') -and + $indentSpaces -le $Cursor.ParentIndent) { + $mark = New-YamlMark -Index $Cursor.Index -Line $Cursor.Line -Column $Cursor.Column + throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidFlowIndentation' -Message ( + 'A multiline flow scalar must be indented beyond its surrounding block context.' + )) } continue } - if ($pendingSpace -and $builder.Length -gt 0) { - [void] $builder.Append(' ') + if ($pendingBreaks -gt 0 -and $builder.Length -gt 0) { + [void] $builder.Append( + $(if ($pendingBreaks -eq 1) { ' ' } else { "`n" * ($pendingBreaks - 1) }) + ) + } elseif ($pendingWhiteSpace.Length -gt 0 -and $builder.Length -gt 0) { + [void] $builder.Append($pendingWhiteSpace) } - $pendingSpace = $false + $pendingBreaks = 0 + $pendingWhiteSpace.Clear() | Out-Null [void] $builder.Append($character) if ($builder.Length -gt $Context.MaxScalarLength) { throw (New-YamlException -Start $start -End $start -ErrorId 'YamlScalarLimitExceeded' -Message ( @@ -580,7 +606,7 @@ function Read-YamlFlowNode { $lastContentLine = $Cursor.Line Move-YamlCursor -Cursor $Cursor -Context $Context } - $value = $builder.ToString().TrimEnd() + $value = $builder.ToString().TrimEnd(' ', "`t") if ([string]::IsNullOrEmpty($value)) { throw (New-YamlException -Start $start -End $start -ErrorId 'YamlExpectedNode' -Message ( 'A YAML node was expected.' diff --git a/src/functions/private/Read-YamlNodeProperty.ps1 b/src/functions/private/Read-YamlNodeProperty.ps1 index 98228d1..fcbd5b5 100644 --- a/src/functions/private/Read-YamlNodeProperty.ps1 +++ b/src/functions/private/Read-YamlNodeProperty.ps1 @@ -25,7 +25,8 @@ function Read-YamlNodeProperty { $unknown = $false $anchor = '' while ($position -lt $Text.Length) { - while ($position -lt $Text.Length -and [char]::IsWhiteSpace($Text[$position])) { + while ($position -lt $Text.Length -and + (Test-YamlWhiteSpace -Character $Text[$position])) { $position++ } if ($position -ge $Text.Length) { @@ -47,7 +48,8 @@ function Read-YamlNodeProperty { } $position = $end + 1 } else { - while ($position -lt $Text.Length -and -not [char]::IsWhiteSpace($Text[$position]) -and + while ($position -lt $Text.Length -and + -not (Test-YamlWhiteSpace -Character $Text[$position]) -and $Text[$position] -notin @(',', '[', ']', '{', '}')) { $position++ } @@ -76,7 +78,8 @@ function Read-YamlNodeProperty { )) } $start = ++$position - while ($position -lt $Text.Length -and -not [char]::IsWhiteSpace($Text[$position]) -and + while ($position -lt $Text.Length -and + -not (Test-YamlWhiteSpace -Character $Text[$position]) -and $Text[$position] -notin @(',', '[', ']', '{', '}')) { $position++ } @@ -97,7 +100,10 @@ function Read-YamlNodeProperty { Tag = $tag HasUnknownTag = $unknown Anchor = $anchor - Rest = $Text.Substring($position).TrimStart() - Consumed = $position + ($Text.Substring($position).Length - $Text.Substring($position).TrimStart().Length) + Rest = $Text.Substring($position).TrimStart(' ', "`t") + Consumed = $position + ( + $Text.Substring($position).Length - + $Text.Substring($position).TrimStart(' ', "`t").Length + ) } } diff --git a/src/functions/private/Read-YamlPlainScalar.ps1 b/src/functions/private/Read-YamlPlainScalar.ps1 index 7602340..cf22818 100644 --- a/src/functions/private/Read-YamlPlainScalar.ps1 +++ b/src/functions/private/Read-YamlPlainScalar.ps1 @@ -35,9 +35,8 @@ function Read-YamlPlainScalar { $start = New-YamlMark -Index ($Context.LineStarts[$startLine] + $FirstColumn) -Line $startLine ` -Column $FirstColumn $parts = [System.Collections.Generic.List[object]]::new() - $firstCommentMatch = [regex]::Match($FirstText, '(?', "'", '"', '%', '@', '`' @@ -45,7 +44,7 @@ function Read-YamlPlainScalar { $conditionalIndicator = $firstCharacter -in @('-', '?', ':') if ($forbiddenFirst -or ( $conditionalIndicator -and ( - $firstValue.Length -eq 1 -or [char]::IsWhiteSpace($firstValue[1]) + $firstValue.Length -eq 1 -or (Test-YamlWhiteSpace -Character $firstValue[1]) ) )) { throw (New-YamlException -Start $start -End $start -ErrorId 'YamlInvalidPlainScalar' -Message ( @@ -86,13 +85,12 @@ function Read-YamlPlainScalar { break } $sourceContent = $line.Substring($indent) - $commentMatch = [regex]::Match($sourceContent, '(? + [CmdletBinding()] + [OutputType([bool])] + param ( + [Parameter(Mandatory)] + [string] $Text, + + [Parameter(Mandatory)] + [ValidateRange(0, 2147483647)] + [int] $Index, + + [switch] $Flow + ) + + if ($Index -ge $Text.Length -or $Text[$Index] -cne ':') { + return $false + } + if ($Index + 1 -ge $Text.Length) { + return $true + } + + $next = $Text[$Index + 1] + if ((Test-YamlWhiteSpace -Character $next) -or $next -ceq "`n") { + return $true + } + return $Flow -and $next -in @(',', '[', ']', '{', '}') +} diff --git a/src/functions/private/Test-YamlTagUriText.ps1 b/src/functions/private/Test-YamlTagUriText.ps1 index 6459d39..216139b 100644 --- a/src/functions/private/Test-YamlTagUriText.ps1 +++ b/src/functions/private/Test-YamlTagUriText.ps1 @@ -19,7 +19,8 @@ function Test-YamlTagUriText { for ($index = 0; $index -lt $Text.Length; $index++) { $character = $Text[$index] - if ([char]::IsWhiteSpace($character) -or [char]::IsControl($character) -or + if ((Test-YamlWhiteSpace -Character $character) -or $character -ceq "`n" -or + [char]::IsControl($character) -or $character -in @('<', '>', '{', '}')) { return $false } diff --git a/src/functions/private/Test-YamlWhiteSpace.ps1 b/src/functions/private/Test-YamlWhiteSpace.ps1 new file mode 100644 index 0000000..39ac42d --- /dev/null +++ b/src/functions/private/Test-YamlWhiteSpace.ps1 @@ -0,0 +1,14 @@ +function Test-YamlWhiteSpace { + <# + .SYNOPSIS + Tests the YAML s-white production. + #> + [CmdletBinding()] + [OutputType([bool])] + param ( + [Parameter(Mandatory)] + [char] $Character + ) + + return $Character -ceq ' ' -or $Character -ceq "`t" +} diff --git a/tests/ConvertFrom-Yaml.Tests.ps1 b/tests/ConvertFrom-Yaml.Tests.ps1 index 2b0842a..7c38742 100644 --- a/tests/ConvertFrom-Yaml.Tests.ps1 +++ b/tests/ConvertFrom-Yaml.Tests.ps1 @@ -137,6 +137,22 @@ folded: > , $result | Should -BeOfType [object[]] $result | Should -Be @('one', 'two') } + + It 'treats only YAML s-white as structural whitespace' { + $nbsp = [char] 0x00A0 + + ("key:${nbsp}value" | ConvertFrom-Yaml) | Should -Be "key:${nbsp}value" + ("-${nbsp}item" | ConvertFrom-Yaml) | Should -Be "-${nbsp}item" + + $flow = "[foo${nbsp}bar]" | ConvertFrom-Yaml -NoEnumerate + $flow | Should -Be @("foo${nbsp}bar") + } + + It 'preserves flow scalar spaces and folds flow line breaks' { + $result = "[foo bar, foo`n`n bar]" | ConvertFrom-Yaml -NoEnumerate + + $result | Should -Be @('foo bar', "foo`nbar") + } } Context 'Streams and pipeline input' { @@ -156,9 +172,33 @@ folded: > $result[0].name | Should -Be 'first' $result[1].name | Should -Be 'second' } + + It 'consumes byte order marks only at legal document boundaries' { + $bom = [char] 0xFEFF + $documents = @( + "${bom}%YAML 1.2`n---`none`n...`n${bom}---`ntwo" | + ConvertFrom-Yaml + ) + + $documents | Should -Be @('one', 'two') + ("${bom}---`nvalue" | ConvertFrom-Yaml) | Should -Be 'value' + ("foo${bom}bar" | Test-Yaml) | Should -BeFalse + ("---`n${bom}value" | Test-Yaml) | Should -BeFalse + } } Context 'Tags, anchors, and aliases' { + It 'stops anchor and alias names before mapping indicators' { + $emptyKey = '&a: value' | ConvertFrom-Yaml -AsHashtable + $aliasedKey = '{anchor: &a foo, *a: value}' | + ConvertFrom-Yaml -AsHashtable + + $emptyKey.Count | Should -Be 1 + $emptyKey[[System.DBNull]::Value] | Should -Be 'value' + @($aliasedKey.Keys) | Should -Be @('anchor', 'foo') + @($aliasedKey.Values) | Should -Be @('foo', 'value') + } + It 'constructs explicit standard scalar tags safely' { $result = @' text: !!str 42 From 1af06a98007581dc57d2e8e4bd70cf85fc90e33c Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 23 Jul 2026 23:05:01 +0200 Subject: [PATCH 02/24] Count implicit keys by Unicode scalar Apply the 1024-character limit with System.Text.Rune, cover plain and quoted boundaries, and retain the official multiline flow-mapping grammar while rejecting multiline flow-sequence pairs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../private/Get-YamlImplicitKeyLength.ps1 | 22 +++++++++++++ src/functions/private/Get-YamlRuneCount.ps1 | 22 +++++++++++++ .../private/Read-YamlBlockMapping.ps1 | 15 +++++---- src/functions/private/Read-YamlFlowNode.ps1 | 10 +++--- tests/ConvertFrom-Yaml.Tests.ps1 | 11 +++++++ tests/Test-Yaml.Tests.ps1 | 31 +++++++++++++++++++ 6 files changed, 99 insertions(+), 12 deletions(-) create mode 100644 src/functions/private/Get-YamlImplicitKeyLength.ps1 create mode 100644 src/functions/private/Get-YamlRuneCount.ps1 diff --git a/src/functions/private/Get-YamlImplicitKeyLength.ps1 b/src/functions/private/Get-YamlImplicitKeyLength.ps1 new file mode 100644 index 0000000..ad583b4 --- /dev/null +++ b/src/functions/private/Get-YamlImplicitKeyLength.ps1 @@ -0,0 +1,22 @@ +function Get-YamlImplicitKeyLength { + <# + .SYNOPSIS + Gets an implicit key length in Unicode scalar values. + #> + [CmdletBinding()] + [OutputType([int])] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Node, + + [Parameter(Mandatory)] + [pscustomobject] $Context + ) + + if ($Node.Kind.Equals('Scalar', [System.StringComparison]::Ordinal)) { + return Get-YamlRuneCount -Text ([string] $Node.Value) + } + + $sourceLength = [Math]::Max(0, $Node.End.Index - $Node.Start.Index) + return Get-YamlRuneCount -Text $Context.Text.Substring($Node.Start.Index, $sourceLength) +} diff --git a/src/functions/private/Get-YamlRuneCount.ps1 b/src/functions/private/Get-YamlRuneCount.ps1 new file mode 100644 index 0000000..dd838c0 --- /dev/null +++ b/src/functions/private/Get-YamlRuneCount.ps1 @@ -0,0 +1,22 @@ +function Get-YamlRuneCount { + <# + .SYNOPSIS + Counts Unicode scalar values in validated YAML text. + #> + [CmdletBinding()] + [OutputType([int])] + param ( + [Parameter(Mandatory)] + [AllowEmptyString()] + [string] $Text + ) + + $count = 0 + $index = 0 + while ($index -lt $Text.Length) { + $rune = [System.Text.Rune]::GetRuneAt($Text, $index) + $index += $rune.Utf16SequenceLength + $count++ + } + return $count +} diff --git a/src/functions/private/Read-YamlBlockMapping.ps1 b/src/functions/private/Read-YamlBlockMapping.ps1 index 8e8177b..6eb3d25 100644 --- a/src/functions/private/Read-YamlBlockMapping.ps1 +++ b/src/functions/private/Read-YamlBlockMapping.ps1 @@ -182,14 +182,6 @@ function Read-YamlBlockMapping { 'A block mapping entry is missing its value indicator.' )) } - if ($colon -gt 1024) { - $mark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $contentColumn + $colon) ` - -Line $lineNumber -Column ($contentColumn + $colon) - throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidImplicitKey' -Message ( - 'An implicit mapping key cannot exceed 1024 characters.' - )) - } - if ($colon -eq 0) { $mark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $contentColumn) ` -Line $lineNumber -Column $contentColumn @@ -199,6 +191,13 @@ function Read-YamlBlockMapping { $key = Read-YamlBlockKey -Context $Context -Text $keyText -Line $lineNumber ` -Column $contentColumn -Depth ($Depth + 1) } + if ((Get-YamlImplicitKeyLength -Node $key -Context $Context) -gt 1024) { + $mark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $contentColumn + $colon) ` + -Line $lineNumber -Column ($contentColumn + $colon) + throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidImplicitKey' -Message ( + 'An implicit mapping key cannot exceed 1024 Unicode scalar values.' + )) + } $valueStart = $colon + 1 $valueSource = $content.Substring($valueStart) diff --git a/src/functions/private/Read-YamlFlowNode.ps1 b/src/functions/private/Read-YamlFlowNode.ps1 index 7a74b76..021328e 100644 --- a/src/functions/private/Read-YamlFlowNode.ps1 +++ b/src/functions/private/Read-YamlFlowNode.ps1 @@ -190,11 +190,11 @@ function Read-YamlFlowNode { if ($Cursor.Index -lt $Context.Text.Length -and $Context.Text[$Cursor.Index] -eq ':') { if (-not $explicitPair -and ( $Cursor.Line -ne $itemStartLine -or - $Cursor.Index - $item.Start.Index -gt 1024 + (Get-YamlImplicitKeyLength -Node $item -Context $Context) -gt 1024 )) { $mark = New-YamlMark -Index $Cursor.Index -Line $Cursor.Line -Column $Cursor.Column throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidImplicitKey' -Message ( - 'An implicit mapping key must fit on one line and cannot exceed 1024 characters.' + 'An implicit mapping key must fit on one line and cannot exceed 1024 Unicode scalar values.' )) } $pair = New-YamlSyntaxNode -Context $Context -Kind Mapping -Depth ($Depth + 1) ` @@ -269,10 +269,12 @@ function Read-YamlFlowNode { } } Skip-YamlFlowTrivia -Cursor $Cursor -Context $Context - if (-not $explicit -and $Cursor.Index - $key.Start.Index -gt 1024) { + if (-not $explicit -and ( + (Get-YamlImplicitKeyLength -Node $key -Context $Context) -gt 1024 + )) { $mark = New-YamlMark -Index $Cursor.Index -Line $Cursor.Line -Column $Cursor.Column throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidImplicitKey' -Message ( - 'An implicit mapping key must fit on one line and cannot exceed 1024 characters.' + 'An implicit mapping key must fit on one line and cannot exceed 1024 Unicode scalar values.' )) } if ($Cursor.Index -ge $Context.Text.Length -or $Context.Text[$Cursor.Index] -ne ':') { diff --git a/tests/ConvertFrom-Yaml.Tests.ps1 b/tests/ConvertFrom-Yaml.Tests.ps1 index 7c38742..34527fc 100644 --- a/tests/ConvertFrom-Yaml.Tests.ps1 +++ b/tests/ConvertFrom-Yaml.Tests.ps1 @@ -153,6 +153,17 @@ folded: > $result | Should -Be @('foo bar', "foo`nbar") } + + It 'accepts adjacent flow collection values after a colon' { + $sequenceValue = '{foo:[bar]}' | ConvertFrom-Yaml + $mappingValue = '{foo:{bar: baz}}' | ConvertFrom-Yaml + $sequencePair = '[foo:[bar]]' | ConvertFrom-Yaml -NoEnumerate + + $sequenceValue.foo | Should -Be @('bar') + $mappingValue.foo.bar | Should -Be 'baz' + $sequencePair.Count | Should -Be 1 + $sequencePair[0].foo | Should -Be @('bar') + } } Context 'Streams and pipeline input' { diff --git a/tests/Test-Yaml.Tests.ps1 b/tests/Test-Yaml.Tests.ps1 index af2b143..9e4cb9d 100644 --- a/tests/Test-Yaml.Tests.ps1 +++ b/tests/Test-Yaml.Tests.ps1 @@ -37,6 +37,37 @@ Describe 'Test-Yaml' { ("a: &a value`nb: !foo *a" | Test-Yaml) | Should -BeFalse } + It 'enforces implicit-key line and Unicode scalar limits' { + $emoji = [char]::ConvertFromUtf32(0x1F600) + + ("['multi`n line': value]" | Test-Yaml) | Should -BeFalse + ("[multi`n line: value]" | Test-Yaml) | Should -BeFalse + + foreach ($quoted in @($false, $true)) { + foreach ($length in @(1024, 1025)) { + $text = 'k' * $length + $key = if ($quoted) { "'$text'" } else { $text } + foreach ($yaml in @( + "[$key`: value]", + "{$key`: value}" + )) { + ($yaml | Test-Yaml) | Should -Be ($length -eq 1024) + } + } + } + + foreach ($length in @(513, 1024, 1025)) { + $key = $emoji * $length + ("{$key`: value}" | Test-Yaml) | + Should -Be ($length -le 1024) + } + } + + It 'accepts multiline keys in flow mappings' { + ("{'multi`n line': value}" | Test-Yaml) | Should -BeTrue + ("{multi`n line: value}" | Test-Yaml) | Should -BeTrue + } + It 'validates tag directives, tag tokens, and alias properties' { ("%TAG !! tag:example.com,2000:app/`n---`n!!int value" | Test-Yaml) | Should -BeTrue From 29797c7ee54246dc399f37dc0ce31c1846e2989a Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 23 Jul 2026 23:08:45 +0200 Subject: [PATCH 03/24] Enforce YAML tag grammar and identity Include effective tags in structural key fingerprints, validate ns-uri-char and percent escapes by whitelist, and accept reserved directives through their YAML productions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../private/Get-YamlEffectiveTag.ps1 | 46 +++++++++++++++++++ .../Get-YamlEmissionNodeFingerprint.ps1 | 20 ++++++-- .../private/Get-YamlNodeFingerprint.ps1 | 33 ++++++------- .../private/Read-YamlDirectiveBlock.ps1 | 9 ++-- src/functions/private/Resolve-YamlTag.ps1 | 4 +- .../private/Test-YamlReservedDirective.ps1 | 40 ++++++++++++++++ src/functions/private/Test-YamlTagUriText.ps1 | 40 +++++++++++----- tests/Test-Yaml.Tests.ps1 | 25 ++++++++++ 8 files changed, 179 insertions(+), 38 deletions(-) create mode 100644 src/functions/private/Get-YamlEffectiveTag.ps1 create mode 100644 src/functions/private/Test-YamlReservedDirective.ps1 diff --git a/src/functions/private/Get-YamlEffectiveTag.ps1 b/src/functions/private/Get-YamlEffectiveTag.ps1 new file mode 100644 index 0000000..ea70947 --- /dev/null +++ b/src/functions/private/Get-YamlEffectiveTag.ps1 @@ -0,0 +1,46 @@ +function Get-YamlEffectiveTag { + <# + .SYNOPSIS + Gets the effective representation tag for a YAML node. + #> + [CmdletBinding()] + [OutputType([string])] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Node, + + [Parameter(Mandatory)] + [AllowNull()] + [object] $Value + ) + + if (-not [string]::IsNullOrEmpty($Node.Tag)) { + return $Node.Tag + } + if ($Node.Kind.Equals('Sequence', [System.StringComparison]::Ordinal)) { + return 'tag:yaml.org,2002:seq' + } + if ($Node.Kind.Equals('Mapping', [System.StringComparison]::Ordinal)) { + return 'tag:yaml.org,2002:map' + } + + if ($null -eq $Value) { + return 'tag:yaml.org,2002:null' + } + if ($Value -is [string]) { + return 'tag:yaml.org,2002:str' + } + if ($Value -is [bool]) { + return 'tag:yaml.org,2002:bool' + } + if ($Value -is [byte[]]) { + return 'tag:yaml.org,2002:binary' + } + if ($Value -is [datetime] -or $Value -is [datetimeoffset]) { + return 'tag:yaml.org,2002:timestamp' + } + if ($Value -is [decimal] -or $Value -is [double] -or $Value -is [single]) { + return 'tag:yaml.org,2002:float' + } + return 'tag:yaml.org,2002:int' +} diff --git a/src/functions/private/Get-YamlEmissionNodeFingerprint.ps1 b/src/functions/private/Get-YamlEmissionNodeFingerprint.ps1 index f69f9e8..a3b176b 100644 --- a/src/functions/private/Get-YamlEmissionNodeFingerprint.ps1 +++ b/src/functions/private/Get-YamlEmissionNodeFingerprint.ps1 @@ -65,7 +65,11 @@ function Get-YamlEmissionNodeFingerprint { ResolvedValue = $null MaxNumericLength = 1048576 }) - $fingerprint = Get-YamlScalarFingerprint -Value $resolved.Value -Hasher $Hasher + $valueFingerprint = Get-YamlScalarFingerprint -Value $resolved.Value -Hasher $Hasher + $effectiveTag = Get-YamlEffectiveTag -Node $frame.Node -Value $resolved.Value + $fingerprint = Get-YamlFingerprintHash -Value ( + 'scalar:{0}:{1}:{2}' -f $effectiveTag.Length, $effectiveTag, $valueFingerprint + ) -Hasher $Hasher if ($hasReferenceId) { $Cache[$frame.Node.ReferenceId] = $fingerprint [void] $Active.Remove($frame.Node.ReferenceId) @@ -86,8 +90,13 @@ function Get-YamlEmissionNodeFingerprint { if ($frame.State -eq 'Sequence') { if ($frame.Index -ge $frame.Node.Items.Count) { + $effectiveTag = Get-YamlEffectiveTag -Node $frame.Node -Value $null $fingerprint = Get-YamlFingerprintHash -Value ( - 'sequence:{0}' -f ($frame.Parts -join '|') + 'sequence:{0}:{1}:{2}' -f @( + $effectiveTag.Length, + $effectiveTag, + ($frame.Parts -join '|') + ) ) -Hasher $Hasher if ($frame.Node.ReferenceId -ne 0) { $Cache[$frame.Node.ReferenceId] = $fingerprint @@ -120,8 +129,13 @@ function Get-YamlEmissionNodeFingerprint { if ($frame.State -eq 'MappingKey') { if ($frame.Index -ge $frame.Node.Entries.Count) { $frame.Parts.Sort([System.StringComparer]::Ordinal) + $effectiveTag = Get-YamlEffectiveTag -Node $frame.Node -Value $null $fingerprint = Get-YamlFingerprintHash -Value ( - 'mapping:{0}' -f ($frame.Parts -join '|') + 'mapping:{0}:{1}:{2}' -f @( + $effectiveTag.Length, + $effectiveTag, + ($frame.Parts -join '|') + ) ) -Hasher $Hasher if ($frame.Node.ReferenceId -ne 0) { $Cache[$frame.Node.ReferenceId] = $fingerprint diff --git a/src/functions/private/Get-YamlNodeFingerprint.ps1 b/src/functions/private/Get-YamlNodeFingerprint.ps1 index 9206142..37ca9d9 100644 --- a/src/functions/private/Get-YamlNodeFingerprint.ps1 +++ b/src/functions/private/Get-YamlNodeFingerprint.ps1 @@ -57,7 +57,11 @@ function Get-YamlNodeFingerprint { if ($effective.Kind -eq 'Scalar') { $resolved = Resolve-YamlScalar -Node $effective - $fingerprint = Get-YamlScalarFingerprint -Value $resolved.Value -Hasher $Hasher + $valueFingerprint = Get-YamlScalarFingerprint -Value $resolved.Value -Hasher $Hasher + $effectiveTag = Get-YamlEffectiveTag -Node $effective -Value $resolved.Value + $fingerprint = Get-YamlFingerprintHash -Value ( + 'scalar:{0}:{1}:{2}' -f $effectiveTag.Length, $effectiveTag, $valueFingerprint + ) -Hasher $Hasher $Cache[$effective.Id] = $fingerprint [void] $Active.Remove($effective.Id) $frame.Holder.Value = $fingerprint @@ -76,15 +80,12 @@ function Get-YamlNodeFingerprint { if ($frame.State -eq 'Sequence') { if ($frame.Index -ge $frame.Node.Items.Count) { - $semanticTag = if ( - $frame.Node.Tag -ceq 'tag:yaml.org,2002:omap' -or - $frame.Node.Tag -ceq 'tag:yaml.org,2002:pairs' - ) { - $frame.Node.Tag - } else { - 'tag:yaml.org,2002:seq' - } - $canonical = 'sequence:{0}:{1}' -f $semanticTag, ($frame.Parts -join '|') + $semanticTag = Get-YamlEffectiveTag -Node $frame.Node -Value $null + $canonical = 'sequence:{0}:{1}:{2}' -f @( + $semanticTag.Length, + $semanticTag, + ($frame.Parts -join '|') + ) $fingerprint = Get-YamlFingerprintHash -Value $canonical -Hasher $Hasher $Cache[$frame.Node.Id] = $fingerprint [void] $Active.Remove($frame.Node.Id) @@ -115,12 +116,12 @@ function Get-YamlNodeFingerprint { if ($frame.State -eq 'MappingKey') { if ($frame.Index -ge $frame.Node.Entries.Count) { $frame.Parts.Sort([System.StringComparer]::Ordinal) - $mappingTag = if ($frame.Node.Tag -ceq 'tag:yaml.org,2002:set') { - $frame.Node.Tag - } else { - 'tag:yaml.org,2002:map' - } - $canonical = 'mapping:{0}:{1}' -f $mappingTag, ($frame.Parts -join '|') + $mappingTag = Get-YamlEffectiveTag -Node $frame.Node -Value $null + $canonical = 'mapping:{0}:{1}:{2}' -f @( + $mappingTag.Length, + $mappingTag, + ($frame.Parts -join '|') + ) $fingerprint = Get-YamlFingerprintHash -Value $canonical -Hasher $Hasher $Cache[$frame.Node.Id] = $fingerprint [void] $Active.Remove($frame.Node.Id) diff --git a/src/functions/private/Read-YamlDirectiveBlock.ps1 b/src/functions/private/Read-YamlDirectiveBlock.ps1 index 37853d1..627be01 100644 --- a/src/functions/private/Read-YamlDirectiveBlock.ps1 +++ b/src/functions/private/Read-YamlDirectiveBlock.ps1 @@ -33,7 +33,7 @@ function Read-YamlDirectiveBlock { if ($directive -cmatch '^%YAML(?:[ \t]|$)') { if ($yamlDirectiveSeen -or $directive -cnotmatch ( '^%YAML[ \t]+([0-9]+)\.([0-9]+)(?:[ \t]+#.*)?$' - ) -or $Matches[1] -ne '1') { + ) -or -not $Matches[1].Equals('1', [System.StringComparison]::Ordinal)) { throw (New-YamlException -Start $mark -End $mark ` -ErrorId 'YamlInvalidDirective' -Message ( "The YAML directive '$directive' is malformed, duplicated, or requests an unsupported version." @@ -63,16 +63,15 @@ function Read-YamlDirectiveBlock { "A YAML tag prefix exceeds the configured limit of $($Context.MaxTagLength) characters." )) } - if ($prefix -ne '!' -and -not (Test-YamlTagUriText -Text $prefix)) { + if (-not $prefix.Equals('!', [System.StringComparison]::Ordinal) -and + -not (Test-YamlTagUriText -Text $prefix)) { throw (New-YamlException -Start $mark -End $mark ` -ErrorId 'YamlInvalidDirective' -Message ( "The TAG directive prefix '$prefix' contains invalid URI text." )) } $tagHandles[$handle] = $prefix - } elseif ($directive -cnotmatch ( - '^%[A-Za-z0-9]+(?:[ \t]+[^# \t][^#]*?)?(?:[ \t]+#.*)?$' - )) { + } elseif (-not (Test-YamlReservedDirective -Directive $directive)) { throw (New-YamlException -Start $mark -End $mark ` -ErrorId 'YamlInvalidDirective' -Message ( "The directive '$directive' is malformed." diff --git a/src/functions/private/Resolve-YamlTag.ps1 b/src/functions/private/Resolve-YamlTag.ps1 index 6c5893d..7483f69 100644 --- a/src/functions/private/Resolve-YamlTag.ps1 +++ b/src/functions/private/Resolve-YamlTag.ps1 @@ -48,7 +48,7 @@ function Resolve-YamlTag { $suffix = $Token.Substring($secondBang + 1) } } - if ($Token -eq '!') { + if ($Token.Equals('!', [System.StringComparison]::Ordinal)) { $prefix = '' $suffix = '!' } elseif ([string]::IsNullOrEmpty($suffix) -or @@ -62,7 +62,7 @@ function Resolve-YamlTag { "The tag handle '$handle' was not declared." )) } - if ($Token -ne '!') { + if (-not $Token.Equals('!', [System.StringComparison]::Ordinal)) { $prefix = $Context.TagHandles[$handle] } } diff --git a/src/functions/private/Test-YamlReservedDirective.ps1 b/src/functions/private/Test-YamlReservedDirective.ps1 new file mode 100644 index 0000000..47880c1 --- /dev/null +++ b/src/functions/private/Test-YamlReservedDirective.ps1 @@ -0,0 +1,40 @@ +function Test-YamlReservedDirective { + <# + .SYNOPSIS + Tests the YAML reserved-directive name and parameter productions. + #> + [CmdletBinding()] + [OutputType([bool])] + param ( + [Parameter(Mandatory)] + [string] $Directive + ) + + if ($Directive.Length -lt 2 -or -not $Directive[0].Equals([char] '%')) { + return $false + } + + $comment = Find-YamlCommentStart -Text $Directive + $contentEnd = if ($comment -ge 0) { $comment } else { $Directive.Length } + $body = $Directive.Substring(1, $contentEnd - 1).TrimEnd(' ', "`t") + if ($body.Length -eq 0 -or (Test-YamlWhiteSpace -Character $body[0])) { + return $false + } + + $index = 0 + while ($index -lt $body.Length) { + $tokenStart = $index + while ($index -lt $body.Length -and + -not (Test-YamlWhiteSpace -Character $body[$index])) { + $index++ + } + if ($index -eq $tokenStart) { + return $false + } + while ($index -lt $body.Length -and + (Test-YamlWhiteSpace -Character $body[$index])) { + $index++ + } + } + return $true +} diff --git a/src/functions/private/Test-YamlTagUriText.ps1 b/src/functions/private/Test-YamlTagUriText.ps1 index 216139b..9a8a546 100644 --- a/src/functions/private/Test-YamlTagUriText.ps1 +++ b/src/functions/private/Test-YamlTagUriText.ps1 @@ -17,24 +17,40 @@ function Test-YamlTagUriText { return $false } + $uriPunctuation = '#;/?:@&=+$,_.!~*''()[]' + $shorthandExcluded = '!,[]' for ($index = 0; $index -lt $Text.Length; $index++) { $character = $Text[$index] - if ((Test-YamlWhiteSpace -Character $character) -or $character -ceq "`n" -or - [char]::IsControl($character) -or - $character -in @('<', '>', '{', '}')) { - return $false - } - if ($Shorthand -and $character -in @('!', '[', ']', ',')) { - return $false - } - if ($character -ne '%') { + if (-not $character.Equals([char] '%')) { + $code = [int] $character + $isWordCharacter = ( + ($code -ge 0x30 -and $code -le 0x39) -or + ($code -ge 0x41 -and $code -le 0x5A) -or + ($code -ge 0x61 -and $code -le 0x7A) -or + $character.Equals([char] '-') + ) + if (-not $isWordCharacter -and $uriPunctuation.IndexOf($character) -lt 0) { + return $false + } + if ($Shorthand -and $shorthandExcluded.IndexOf($character) -ge 0) { + return $false + } continue } - if ($index + 2 -ge $Text.Length -or - $Text[$index + 1] -notmatch '^[0-9A-Fa-f]$' -or - $Text[$index + 2] -notmatch '^[0-9A-Fa-f]$') { + + if ($index + 2 -ge $Text.Length) { return $false } + foreach ($offset in 1, 2) { + $hexCode = [int] $Text[$index + $offset] + if (-not ( + ($hexCode -ge 0x30 -and $hexCode -le 0x39) -or + ($hexCode -ge 0x41 -and $hexCode -le 0x46) -or + ($hexCode -ge 0x61 -and $hexCode -le 0x66) + )) { + return $false + } + } $index += 2 } return $true diff --git a/tests/Test-Yaml.Tests.ps1 b/tests/Test-Yaml.Tests.ps1 index 9e4cb9d..460d1f2 100644 --- a/tests/Test-Yaml.Tests.ps1 +++ b/tests/Test-Yaml.Tests.ps1 @@ -71,11 +71,19 @@ Describe 'Test-Yaml' { It 'validates tag directives, tag tokens, and alias properties' { ("%TAG !! tag:example.com,2000:app/`n---`n!!int value" | Test-Yaml) | Should -BeTrue + ("%FOO-BAR baz`n---`nvalue" | Test-Yaml) | Should -BeTrue + ("%FOO:B alpha-beta p#q`n---`nvalue" | Test-Yaml) | Should -BeTrue ("%TAG ! tag:first/`n%TAG ! tag:second/`n---`n!value data" | Test-Yaml) | Should -BeFalse + ("% FOO baz`n---`nvalue" | Test-Yaml) | Should -BeFalse + ("%`n---`nvalue" | Test-Yaml) | Should -BeFalse ('!foo%GG value' | Test-Yaml) | Should -BeFalse + ('!foo\bar value' | Test-Yaml) | Should -BeFalse + ('! value' | Test-Yaml) | Should -BeFalse ('!! value' | Test-Yaml) | Should -BeFalse ('! value' | Test-Yaml) | Should -BeFalse + ('! value' | + Test-Yaml) | Should -BeTrue ("a: &a value`nb: &b *a" | Test-Yaml) | Should -BeFalse } @@ -89,6 +97,23 @@ Describe 'Test-Yaml' { ("1: one`n01: two" | Test-Yaml) | Should -BeFalse } + It 'includes effective tags in mapping-key equality' { + ("!foo x: one`n!bar x: two" | Test-Yaml) | Should -BeTrue + ("!foo x: one`n!foo x: two" | Test-Yaml) | Should -BeFalse + (@' +? !foo [x] +: one +? !bar [x] +: two +'@ | Test-Yaml) | Should -BeTrue + (@' +? !foo {x: y} +: one +? !bar {x: y} +: two +'@ | Test-Yaml) | Should -BeTrue + } + It 'accepts complex keys even though default object projection cannot' { ("? [a, b]`n: value" | Test-Yaml) | Should -BeTrue } From bf412796fbd2fffb4e9b7e85ec8875d17f42a10b Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 23 Jul 2026 23:11:52 +0200 Subject: [PATCH 04/24] Normalize YAML floating key equality Canonicalize Decimal, Double, and Single values by decimal significand and exponent, making large equivalent values and signed zero obey YAML representation equality. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../private/Get-YamlNormalizedFloat.ps1 | 95 +++++++++++++++++++ .../private/Get-YamlScalarFingerprint.ps1 | 28 +----- tests/ConvertFrom-Yaml.Tests.ps1 | 22 +++++ tests/ConvertTo-Yaml.Tests.ps1 | 27 ++++++ 4 files changed, 146 insertions(+), 26 deletions(-) create mode 100644 src/functions/private/Get-YamlNormalizedFloat.ps1 diff --git a/src/functions/private/Get-YamlNormalizedFloat.ps1 b/src/functions/private/Get-YamlNormalizedFloat.ps1 new file mode 100644 index 0000000..f52e142 --- /dev/null +++ b/src/functions/private/Get-YamlNormalizedFloat.ps1 @@ -0,0 +1,95 @@ +function Get-YamlNormalizedFloat { + <# + .SYNOPSIS + Normalizes a finite CLR floating-point value to a decimal significand and exponent. + #> + [CmdletBinding()] + [OutputType([string])] + param ( + [Parameter(Mandatory)] + [object] $Value + ) + + $culture = [System.Globalization.CultureInfo]::InvariantCulture + if ($Value -is [decimal]) { + if ($Value -eq 0) { + return '0e0' + } + $text = $Value.ToString('G29', $culture) + } elseif ($Value -is [single]) { + $number = [single] $Value + if ([single]::IsNaN($number)) { + return 'nan' + } + if ([single]::IsPositiveInfinity($number)) { + return '+inf' + } + if ([single]::IsNegativeInfinity($number)) { + return '-inf' + } + if ($number -eq 0) { + return '0e0' + } + $text = $number.ToString('R', $culture) + } elseif ($Value -is [double]) { + $number = [double] $Value + if ([double]::IsNaN($number)) { + return 'nan' + } + if ([double]::IsPositiveInfinity($number)) { + return '+inf' + } + if ([double]::IsNegativeInfinity($number)) { + return '-inf' + } + if ($number -eq 0) { + return '0e0' + } + $text = $number.ToString('R', $culture) + } else { + throw [System.ArgumentException]::new( + "Type '$($Value.GetType().FullName)' is not a supported floating-point type.", + 'Value' + ) + } + + $match = [regex]::Match( + $text, + '^(?[-+]?)(?[0-9]+)(?:\.(?[0-9]*))?(?:[eE](?[-+]?[0-9]+))?$', + [System.Text.RegularExpressions.RegexOptions]::CultureInvariant + ) + if (-not $match.Success) { + throw [System.InvalidOperationException]::new( + "The invariant floating-point text '$text' could not be normalized." + ) + } + + $fraction = $match.Groups['fraction'].Value + $digits = ($match.Groups['integer'].Value + $fraction).TrimStart('0') + if ($digits.Length -eq 0) { + return '0e0' + } + + $exponent = 0 - $fraction.Length + if ($match.Groups['exponent'].Success) { + $exponent += [int]::Parse( + $match.Groups['exponent'].Value, + [System.Globalization.NumberStyles]::AllowLeadingSign, + $culture + ) + } + while ($digits.Length -gt 1 -and $digits[$digits.Length - 1].Equals([char] '0')) { + $digits = $digits.Substring(0, $digits.Length - 1) + $exponent++ + } + + $sign = if ($match.Groups['sign'].Value.Equals( + '-', + [System.StringComparison]::Ordinal + )) { + '-' + } else { + '' + } + return '{0}{1}e{2}' -f $sign, $digits, $exponent +} diff --git a/src/functions/private/Get-YamlScalarFingerprint.ps1 b/src/functions/private/Get-YamlScalarFingerprint.ps1 index a438342..ba4c198 100644 --- a/src/functions/private/Get-YamlScalarFingerprint.ps1 +++ b/src/functions/private/Get-YamlScalarFingerprint.ps1 @@ -33,32 +33,8 @@ function Get-YamlScalarFingerprint { [datetime]::SpecifyKind($Value, [System.DateTimeKind]::Utc) } $canonicalValue = 'timestamp:{0}' -f $utcValue.Ticks - } elseif ($Value -is [decimal]) { - if ($Value -eq 0 -and - ([decimal]::GetBits($Value)[3] -band [int]::MinValue) -ne 0) { - $canonicalValue = 'float:-0' - } else { - $canonicalValue = 'float:{0}' -f $Value.ToString( - 'G29', - [cultureinfo]::InvariantCulture - ) - } - } elseif ($Value -is [double]) { - if ([double]::IsNaN($Value)) { - $canonicalValue = 'float:nan' - } elseif ([double]::IsPositiveInfinity($Value)) { - $canonicalValue = 'float:+inf' - } elseif ([double]::IsNegativeInfinity($Value)) { - $canonicalValue = 'float:-inf' - } elseif ($Value -eq 0) { - $negative = [System.BitConverter]::DoubleToInt64Bits([double] $Value) -lt 0 - $canonicalValue = if ($negative) { 'float:-0' } else { 'float:0' } - } else { - $canonicalValue = 'float:{0}' -f $Value.ToString( - 'R', - [cultureinfo]::InvariantCulture - ) - } + } elseif ($Value -is [decimal] -or $Value -is [double] -or $Value -is [single]) { + $canonicalValue = 'float:{0}' -f (Get-YamlNormalizedFloat -Value $Value) } else { $canonicalValue = 'int:{0}' -f $Value.ToString([cultureinfo]::InvariantCulture) } diff --git a/tests/ConvertFrom-Yaml.Tests.ps1 b/tests/ConvertFrom-Yaml.Tests.ps1 index 34527fc..06b9088 100644 --- a/tests/ConvertFrom-Yaml.Tests.ps1 +++ b/tests/ConvertFrom-Yaml.Tests.ps1 @@ -376,6 +376,28 @@ date: !!timestamp 2001-12-14 Should -Throw } + It 'normalizes cross-type finite floats for key equality' { + $yaml = @' +100000000000000000000.0: decimal +1e20: double +'@ + + ($yaml | Test-Yaml) | Should -BeFalse + { $yaml | ConvertFrom-Yaml -AsHashtable } | + Should -Throw -ExpectedMessage '*duplicate mapping key*' + } + + It 'treats signed zero keys as the same YAML representation value' { + $yaml = @' +0.0: positive +-0.0: negative +'@ + + ($yaml | Test-Yaml) | Should -BeFalse + { $yaml | ConvertFrom-Yaml -AsHashtable } | + Should -Throw -ExpectedMessage '*duplicate mapping key*' + } + It 'rejects equivalent offset timestamps as duplicate keys' { $yaml = @' ? !!timestamp 2001-12-15T02:59:43.1Z diff --git a/tests/ConvertTo-Yaml.Tests.ps1 b/tests/ConvertTo-Yaml.Tests.ps1 index 5cc3da6..3c5d7cc 100644 --- a/tests/ConvertTo-Yaml.Tests.ps1 +++ b/tests/ConvertTo-Yaml.Tests.ps1 @@ -181,6 +181,33 @@ Describe 'ConvertTo-Yaml' { Should -Throw -ExpectedMessage '*normalize to the same YAML value*' } + It 'normalizes finite float key fingerprints across CLR types' { + $equal = [System.Collections.Specialized.OrderedDictionary]::new() + $equal.Add( + [decimal]::Parse( + '100000000000000000000.0', + [cultureinfo]::InvariantCulture + ), + 'decimal' + ) + $equal.Add([double] 1e20, 'double') + + { $equal | ConvertTo-Yaml } | + Should -Throw -ExpectedMessage '*normalize to the same YAML value*' + + $different = [System.Collections.Specialized.OrderedDictionary]::new() + $different.Add( + [decimal]::Parse( + '100000000000000000001', + [cultureinfo]::InvariantCulture + ), + 'decimal' + ) + $different.Add([double] 1e20, 'double') + + { $different | ConvertTo-Yaml } | Should -Not -Throw + } + It 'compares unordered complex keys with ordinal case-sensitive sorting' { $firstKey = [System.Collections.Specialized.OrderedDictionary]::new() $firstKey.Add('a', 1) From f973ab4d932483ebbbcc3b1675277a7cc214e328 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 23 Jul 2026 23:15:13 +0200 Subject: [PATCH 05/24] Reject lossy ETS projections Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../private/Add-YamlDictionaryEntry.ps1 | 37 +++++++++++++++++++ .../private/ConvertFrom-YamlNode.ps1 | 18 +++++++-- .../private/Get-YamlSerializationShape.ps1 | 11 +++++- .../private/Test-YamlReservedPropertyName.ps1 | 25 +++++++++++++ tests/ConvertFrom-Yaml.Tests.ps1 | 25 +++++++++++++ tests/ConvertTo-Yaml.Tests.ps1 | 8 ++++ 6 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 src/functions/private/Add-YamlDictionaryEntry.ps1 create mode 100644 src/functions/private/Test-YamlReservedPropertyName.ps1 diff --git a/src/functions/private/Add-YamlDictionaryEntry.ps1 b/src/functions/private/Add-YamlDictionaryEntry.ps1 new file mode 100644 index 0000000..5d53b06 --- /dev/null +++ b/src/functions/private/Add-YamlDictionaryEntry.ps1 @@ -0,0 +1,37 @@ +function Add-YamlDictionaryEntry { + <# + .SYNOPSIS + Adds a projected mapping entry with a YAML-classified collision error. + #> + [Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSUseShouldProcessForStateChangingFunctions', '', + Justification = 'Mutates an internal projection dictionary.' + )] + [CmdletBinding()] + param ( + [Parameter(Mandatory)] + [System.Collections.IDictionary] $Dictionary, + + [Parameter(Mandatory)] + [object] $Key, + + [Parameter(Mandatory)] + [AllowNull()] + [object] $Value, + + [Parameter(Mandatory)] + [pscustomobject] $KeyNode + ) + + try { + $Dictionary.Add($Key, $Value) + } catch [System.Management.Automation.MethodInvocationException] { + if ($_.Exception.InnerException -isnot [System.ArgumentException]) { + throw + } + throw (New-YamlException -Start $KeyNode.Start -End $KeyNode.End ` + -ErrorId 'YamlProjectionKeyCollision' -Message ( + 'Distinct YAML mapping keys cannot be projected distinctly into a PowerShell dictionary.' + )) + } +} diff --git a/src/functions/private/ConvertFrom-YamlNode.ps1 b/src/functions/private/ConvertFrom-YamlNode.ps1 index 88dc837..64b297a 100644 --- a/src/functions/private/ConvertFrom-YamlNode.ps1 +++ b/src/functions/private/ConvertFrom-YamlNode.ps1 @@ -154,7 +154,9 @@ function ConvertFrom-YamlNode { continue } if ($frame.State -eq 'OmapValue') { - $frame.Result.Add($frame.Key, $frame.Child.PSObject.Properties['Value'].Value) + Add-YamlDictionaryEntry -Dictionary $frame.Result -Key $frame.Key ` + -Value $frame.Child.PSObject.Properties['Value'].Value ` + -KeyNode $frame.Node.Items[$frame.Index].Entries[0].Key $frame.Index++ $frame.State = 'OmapKey' continue @@ -188,7 +190,8 @@ function ConvertFrom-YamlNode { $frame.Key = $keyValue } if ($frame.Node.Tag -ceq 'tag:yaml.org,2002:set') { - $frame.Result.Add($frame.Key, $null) + Add-YamlDictionaryEntry -Dictionary $frame.Result -Key $frame.Key -Value $null ` + -KeyNode $frame.Node.Entries[$frame.Index].Key $frame.Index++ $frame.State = 'DictionaryKey' continue @@ -209,7 +212,9 @@ function ConvertFrom-YamlNode { continue } if ($frame.State -eq 'DictionaryValue') { - $frame.Result.Add($frame.Key, $frame.Child.PSObject.Properties['Value'].Value) + Add-YamlDictionaryEntry -Dictionary $frame.Result -Key $frame.Key ` + -Value $frame.Child.PSObject.Properties['Value'].Value ` + -KeyNode $frame.Node.Entries[$frame.Index].Key $frame.Index++ $frame.State = 'DictionaryKey' continue @@ -244,6 +249,13 @@ function ConvertFrom-YamlNode { 'This mapping key cannot be represented as a PSCustomObject property. Use -AsHashtable.' )) } + if (Test-YamlReservedPropertyName -Name $frame.Key) { + $keyNode = $frame.Node.Entries[$frame.Index].Key + throw (New-YamlException -Start $keyNode.Start -End $keyNode.End ` + -ErrorId 'YamlPropertyNameReserved' -Message ( + "The mapping key '$($frame.Key)' is reserved by PowerShell ETS. Use -AsHashtable." + )) + } if (-not $frame.Names.Add($frame.Key)) { $keyNode = $frame.Node.Entries[$frame.Index].Key throw (New-YamlException -Start $keyNode.Start -End $keyNode.End ` diff --git a/src/functions/private/Get-YamlSerializationShape.ps1 b/src/functions/private/Get-YamlSerializationShape.ps1 index 796e62e..fb12c68 100644 --- a/src/functions/private/Get-YamlSerializationShape.ps1 +++ b/src/functions/private/Get-YamlSerializationShape.ps1 @@ -57,13 +57,22 @@ function Get-YamlSerializationShape { -not $isByteArray ) $isCustomObject = $Value -is [System.Management.Automation.PSCustomObject] + $isPurePropertyBag = $isCustomObject -or ( + $dataProperties.Count -gt 0 -and $Value.GetType() -eq [object] + ) if ($dataProperties.Count -gt 0 -and ($isDictionary -or $isSequence -or $isByteArray)) { throw (New-YamlSerializationException -Kind NotSupported ` -ErrorId 'YamlMixedObjectSemantics' -Message ( "Type '$($Value.GetType().FullName)' combines collection data with attached note properties and cannot be represented without loss." )) } - $isPropertyBag = $isCustomObject -or $dataProperties.Count -gt 0 + if ($dataProperties.Count -gt 0 -and -not $isPurePropertyBag) { + throw (New-YamlSerializationException -Kind NotSupported ` + -ErrorId 'YamlMixedObjectSemantics' -Message ( + "Type '$($Value.GetType().FullName)' combines scalar data with attached note properties and cannot be represented without loss." + )) + } + $isPropertyBag = $isPurePropertyBag if (-not $isPropertyBag -and ($Value -is [string] -or $Value -is [char])) { $scalar.Value = [string] $Value diff --git a/src/functions/private/Test-YamlReservedPropertyName.ps1 b/src/functions/private/Test-YamlReservedPropertyName.ps1 new file mode 100644 index 0000000..7e94dad --- /dev/null +++ b/src/functions/private/Test-YamlReservedPropertyName.ps1 @@ -0,0 +1,25 @@ +function Test-YamlReservedPropertyName { + <# + .SYNOPSIS + Tests whether a property name is reserved by PowerShell ETS. + #> + [CmdletBinding()] + [OutputType([bool])] + param ( + [Parameter(Mandatory)] + [string] $Name + ) + + foreach ($reservedName in @( + 'PSObject', + 'PSTypeNames', + 'PSBase', + 'PSAdapted', + 'PSExtended' + )) { + if ($Name.Equals($reservedName, [System.StringComparison]::OrdinalIgnoreCase)) { + return $true + } + } + return $false +} diff --git a/tests/ConvertFrom-Yaml.Tests.ps1 b/tests/ConvertFrom-Yaml.Tests.ps1 index 06b9088..8880b37 100644 --- a/tests/ConvertFrom-Yaml.Tests.ps1 +++ b/tests/ConvertFrom-Yaml.Tests.ps1 @@ -125,6 +125,23 @@ folded: > $result['name'] | Should -Be 'two' } + It 'classifies ETS-reserved property names and preserves them with AsHashtable' { + foreach ($name in @( + 'PSObject', + 'psobject', + 'PSTypeNames', + 'PSBase', + 'PSAdapted', + 'PSExtended' + )) { + { "$name`: value" | ConvertFrom-Yaml } | + Should -Throw -ExpectedMessage '*Use -AsHashtable*' + + $result = "$name`: value" | ConvertFrom-Yaml -AsHashtable + $result[$name] | Should -Be 'value' + } + } + It 'enumerates only top-level sequences by default' { $result = "- one`n- two" | ConvertFrom-Yaml @@ -346,6 +363,14 @@ copy: *source $result['set']['one'] | Should -BeNullOrEmpty } + It 'classifies projection collisions from representation-distinct tagged keys' { + $yaml = "!foo x: one`n!bar x: two" + + ($yaml | Test-Yaml) | Should -BeTrue + { $yaml | ConvertFrom-Yaml -AsHashtable } | + Should -Throw -ExpectedMessage '*cannot be projected distinctly*' + } + It 'uses deterministic UTC semantics for zone-less explicit timestamps' { $result = @' offset: !!timestamp 2001-12-14T21:59:43.1+5:30 diff --git a/tests/ConvertTo-Yaml.Tests.ps1 b/tests/ConvertTo-Yaml.Tests.ps1 index 3c5d7cc..6f11653 100644 --- a/tests/ConvertTo-Yaml.Tests.ps1 +++ b/tests/ConvertTo-Yaml.Tests.ps1 @@ -322,6 +322,14 @@ Describe 'ConvertTo-Yaml' { Should -Throw -ExpectedMessage '*combines collection data with attached note properties*' } + It 'rejects attached note properties on scalar values' { + $value = [psobject] 42 + Add-Member -InputObject $value -MemberType NoteProperty -Name metadata -Value 'lossy' + + { ConvertTo-Yaml -InputObject $value } | + Should -Throw -ExpectedMessage '*combines scalar data with attached note properties*' + } + It 'preserves the sign of IEEE negative zero across serialization' { $negativeZero = [BitConverter]::Int64BitsToDouble([long]::MinValue) $yaml = ConvertTo-Yaml -InputObject $negativeZero From edd0130a8fc67a0a6112b4752ac46dfa4725e5e0 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 23 Jul 2026 23:23:50 +0200 Subject: [PATCH 06/24] Bound serialization resource consumption Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/functions/private/ConvertTo-YamlNode.ps1 | 35 +++-- .../private/Get-YamlSerializationShape.ps1 | 41 ++++- .../private/Read-YamlBoundedEnumerable.ps1 | 61 ++++++++ .../Test-YamlSerializationReference.ps1 | 28 ++++ src/functions/public/ConvertTo-Yaml.ps1 | 23 +++ tests/ConvertTo-Yaml.Tests.ps1 | 143 ++++++++++++++++++ 6 files changed, 310 insertions(+), 21 deletions(-) create mode 100644 src/functions/private/Read-YamlBoundedEnumerable.ps1 create mode 100644 src/functions/private/Test-YamlSerializationReference.ps1 diff --git a/src/functions/private/ConvertTo-YamlNode.ps1 b/src/functions/private/ConvertTo-YamlNode.ps1 index fc333f0..ba3f142 100644 --- a/src/functions/private/ConvertTo-YamlNode.ps1 +++ b/src/functions/private/ConvertTo-YamlNode.ps1 @@ -51,6 +51,25 @@ function ConvertTo-YamlNode { )) } + $isReference = Test-YamlSerializationReference -Value $frame.Value + if ($isReference) { + $firstTime = $false + $frame.ReferenceId = $State.IdGenerator.GetId($frame.Value, [ref] $firstTime) + if (-not $firstTime) { + $State.ReferenceCounts[$frame.ReferenceId]++ + if ($State.Active.Contains($frame.ReferenceId)) { + throw (New-YamlSerializationException -ErrorId 'YamlCycleDetected' -Message ( + "A cycle was detected while serializing type '$($frame.Value.GetType().FullName)'." + )) + } + $frame.Holder.Value = $State.NodesById[$frame.ReferenceId] + [void] $stack.Pop() + continue + } + $State.ReferenceCounts[$frame.ReferenceId] = 1 + $State.ReferenceOrder.Add($frame.ReferenceId) + } + $frame.Shape = Get-YamlSerializationShape -Value $frame.Value -State $State ` -EnumsAsStrings:$EnumsAsStrings if ($frame.Shape.Kind -eq 'Scalar') { @@ -59,22 +78,6 @@ function ConvertTo-YamlNode { continue } - $firstTime = $false - $frame.ReferenceId = $State.IdGenerator.GetId($frame.Value, [ref] $firstTime) - if (-not $firstTime) { - $State.ReferenceCounts[$frame.ReferenceId]++ - if ($State.Active.Contains($frame.ReferenceId)) { - throw (New-YamlSerializationException -ErrorId 'YamlCycleDetected' -Message ( - "A cycle was detected while serializing type '$($frame.Value.GetType().FullName)'." - )) - } - $frame.Holder.Value = $State.NodesById[$frame.ReferenceId] - [void] $stack.Pop() - continue - } - - $State.ReferenceCounts[$frame.ReferenceId] = 1 - $State.ReferenceOrder.Add($frame.ReferenceId) if ($frame.Shape.Kind -eq 'Binary') { $frame.Shape.Node.ReferenceId = $frame.ReferenceId $State.NodesById[$frame.ReferenceId] = $frame.Shape.Node diff --git a/src/functions/private/Get-YamlSerializationShape.ps1 b/src/functions/private/Get-YamlSerializationShape.ps1 index fb12c68..095ec4a 100644 --- a/src/functions/private/Get-YamlSerializationShape.ps1 +++ b/src/functions/private/Get-YamlSerializationShape.ps1 @@ -13,7 +13,9 @@ function Get-YamlSerializationShape { [Parameter(Mandatory)] [pscustomobject] $State, - [switch] $EnumsAsStrings + [switch] $EnumsAsStrings, + + [switch] $InspectOnly ) $scalar = New-YamlEmissionNode -Kind Scalar @@ -189,6 +191,15 @@ function Get-YamlSerializationShape { } if ($isByteArray) { + $base64Length = [long] 4 * [long] [System.Math]::Ceiling($Value.LongLength / 3.0) + if ($base64Length -gt $State.MaxScalarLength) { + throw (New-YamlSerializationException -ErrorId 'YamlScalarLimitExceeded' -Message ( + "A scalar exceeds the configured limit of $($State.MaxScalarLength) characters." + )) + } + if ($InspectOnly) { + return [pscustomobject]@{ Kind = 'Binary'; Node = $null; Values = $null } + } $scalar.Tag = 'tag:yaml.org,2002:binary' $scalar.Value = [System.Convert]::ToBase64String($Value) $scalar.Style = 'DoubleQuoted' @@ -197,16 +208,33 @@ function Get-YamlSerializationShape { } if ($isDictionary -or $isPropertyBag) { + if ($InspectOnly) { + return [pscustomobject]@{ Kind = 'Mapping'; Node = $null; Values = $null } + } $entries = [System.Collections.Generic.List[object]]::new() if ($isDictionary) { - foreach ($entry in $Value.GetEnumerator()) { + $remainingNodes = [System.Math]::Max(0, $State.MaxNodes - $State.NodeCount) + $maximumEntries = [int] [System.Math]::Floor($remainingNodes / 2.0) + $rawEntries = Read-YamlBoundedEnumerable -Value $Value ` + -MaximumItems $maximumEntries -MaxNodes $State.MaxNodes -State $State ` + -DictionaryEntries -EnumsAsStrings:$EnumsAsStrings + foreach ($entry in $rawEntries) { $entries.Add([pscustomobject]@{ Key = [object] $entry.Key Value = [object] $entry.Value }) } } else { + if (($dataProperties.Count * 2) -gt ($State.MaxNodes - $State.NodeCount)) { + throw (New-YamlSerializationException -ErrorId 'YamlNodeLimitExceeded' -Message ( + "The object graph exceeds the configured limit of $($State.MaxNodes) nodes." + )) + } foreach ($property in $dataProperties) { + $null = Get-YamlSerializationShape -Value $property.Name -State $State ` + -EnumsAsStrings:$EnumsAsStrings -InspectOnly + $null = Get-YamlSerializationShape -Value $property.Value -State $State ` + -EnumsAsStrings:$EnumsAsStrings -InspectOnly $entries.Add([pscustomobject]@{ Key = [object] $property.Name Value = [object] $property.Value @@ -220,10 +248,13 @@ function Get-YamlSerializationShape { } } if ($isSequence) { - $items = [System.Collections.Generic.List[object]]::new() - foreach ($item in $Value) { - $items.Add([object] $item) + if ($InspectOnly) { + return [pscustomobject]@{ Kind = 'Sequence'; Node = $null; Values = $null } } + $remainingNodes = [System.Math]::Max(0, $State.MaxNodes - $State.NodeCount) + $items = Read-YamlBoundedEnumerable -Value $Value ` + -MaximumItems $remainingNodes -MaxNodes $State.MaxNodes -State $State ` + -EnumsAsStrings:$EnumsAsStrings return [pscustomobject]@{ Kind = 'Sequence' Node = $null diff --git a/src/functions/private/Read-YamlBoundedEnumerable.ps1 b/src/functions/private/Read-YamlBoundedEnumerable.ps1 new file mode 100644 index 0000000..37875b7 --- /dev/null +++ b/src/functions/private/Read-YamlBoundedEnumerable.ps1 @@ -0,0 +1,61 @@ +function Read-YamlBoundedEnumerable { + <# + .SYNOPSIS + Materializes only as many enumerable items as the node budget permits. + #> + [CmdletBinding()] + [OutputType([System.Collections.Generic.List[object]])] + param ( + [Parameter(Mandatory)] + [System.Collections.IEnumerable] $Value, + + [Parameter(Mandatory)] + [ValidateRange(0, 2147483647)] + [int] $MaximumItems, + + [Parameter(Mandatory)] + [int] $MaxNodes, + + [Parameter(Mandatory)] + [pscustomobject] $State, + + [switch] $DictionaryEntries, + + [switch] $EnumsAsStrings + ) + + if ($Value -is [System.Collections.ICollection] -and + $Value.Count -gt $MaximumItems) { + throw (New-YamlSerializationException -ErrorId 'YamlNodeLimitExceeded' -Message ( + "The object graph exceeds the configured limit of $MaxNodes nodes." + )) + } + + $items = [System.Collections.Generic.List[object]]::new() + $enumerator = $Value.GetEnumerator() + try { + while ($enumerator.MoveNext()) { + if ($items.Count -ge $MaximumItems) { + throw (New-YamlSerializationException -ErrorId 'YamlNodeLimitExceeded' -Message ( + "The object graph exceeds the configured limit of $MaxNodes nodes." + )) + } + $item = [object] $enumerator.Current + if ($DictionaryEntries) { + $null = Get-YamlSerializationShape -Value $item.Key -State $State ` + -EnumsAsStrings:$EnumsAsStrings -InspectOnly + $null = Get-YamlSerializationShape -Value $item.Value -State $State ` + -EnumsAsStrings:$EnumsAsStrings -InspectOnly + } else { + $null = Get-YamlSerializationShape -Value $item -State $State ` + -EnumsAsStrings:$EnumsAsStrings -InspectOnly + } + $items.Add($item) + } + } finally { + if ($enumerator -is [System.IDisposable]) { + $enumerator.Dispose() + } + } + Write-Output -InputObject $items -NoEnumerate +} diff --git a/src/functions/private/Test-YamlSerializationReference.ps1 b/src/functions/private/Test-YamlSerializationReference.ps1 new file mode 100644 index 0000000..59a1fc2 --- /dev/null +++ b/src/functions/private/Test-YamlSerializationReference.ps1 @@ -0,0 +1,28 @@ +function Test-YamlSerializationReference { + <# + .SYNOPSIS + Tests whether a value participates in YAML anchor identity. + #> + [CmdletBinding()] + [OutputType([bool])] + param ( + [Parameter()] + [AllowNull()] + [object] $Value + ) + + if ($null -eq $Value -or $Value -is [System.DBNull]) { + return $false + } + if ($Value -is [byte[]] -or + $Value -is [System.Collections.IDictionary] -or + $Value -is [System.Management.Automation.PSCustomObject]) { + return $true + } + if ($Value -is [System.Collections.IEnumerable] -and + $Value -isnot [string] -and + $Value -isnot [char]) { + return $true + } + return $Value.GetType() -eq [object] +} diff --git a/src/functions/public/ConvertTo-Yaml.ps1 b/src/functions/public/ConvertTo-Yaml.ps1 index cbfee38..f13e828 100644 --- a/src/functions/public/ConvertTo-Yaml.ps1 +++ b/src/functions/public/ConvertTo-Yaml.ps1 @@ -76,8 +76,31 @@ function ConvertTo-Yaml { begin { $values = [System.Collections.Generic.List[object]]::new() + $inspectionState = [pscustomobject]@{ + MaxScalarLength = $MaxScalarLength + MaxNodes = $MaxNodes + NodeCount = 0 + } } process { + try { + $null = Get-YamlSerializationShape -Value $InputObject -State $inspectionState ` + -EnumsAsStrings:$EnumsAsStrings -InspectOnly + if ($values.Count -gt 0 -and ($values.Count + 2) -gt $MaxNodes) { + throw (New-YamlSerializationException -ErrorId 'YamlNodeLimitExceeded' -Message ( + "The object graph exceeds the configured limit of $MaxNodes nodes." + )) + } + } catch [System.NotSupportedException] { + $record = New-YamlErrorRecord -Exception $_.Exception ` + -DefaultErrorId 'YamlUnsupportedType' -Category InvalidType -TargetObject $InputObject + $PSCmdlet.ThrowTerminatingError($record) + } catch [System.InvalidOperationException] { + $record = New-YamlErrorRecord -Exception $_.Exception ` + -DefaultErrorId 'YamlSerializationFailed' -Category InvalidOperation ` + -TargetObject $InputObject + $PSCmdlet.ThrowTerminatingError($record) + } $values.Add($InputObject) } end { diff --git a/tests/ConvertTo-Yaml.Tests.ps1 b/tests/ConvertTo-Yaml.Tests.ps1 index 6f11653..607baa0 100644 --- a/tests/ConvertTo-Yaml.Tests.ps1 +++ b/tests/ConvertTo-Yaml.Tests.ps1 @@ -13,6 +13,95 @@ param() BeforeAll { . (Join-Path $PSScriptRoot 'TestBootstrap.ps1') + if ($null -eq ('YamlTests.InfiniteEnumerable' -as [type])) { + Add-Type -TypeDefinition @' +namespace YamlTests +{ + using System; + using System.Collections; + + public sealed class InfiniteEnumerable : IEnumerable + { + private readonly object value; + public int MoveNextCount { get; private set; } + public int DisposeCount { get; private set; } + + public InfiniteEnumerable() : this(1) { } + public InfiniteEnumerable(object value) { this.value = value; } + public IEnumerator GetEnumerator() { return new Enumerator(this, value); } + + private sealed class Enumerator : IEnumerator, IDisposable + { + private readonly InfiniteEnumerable owner; + private readonly object value; + + public Enumerator(InfiniteEnumerable owner, object value) + { + this.owner = owner; + this.value = value; + } + + public object Current { get { return value; } } + + public bool MoveNext() + { + owner.MoveNextCount++; + return true; + } + + public void Reset() { throw new NotSupportedException(); } + public void Dispose() { owner.DisposeCount++; } + } + } + + public sealed class OneShotEnumerable : IEnumerable + { + private readonly object[] values; + public int GetEnumeratorCount { get; private set; } + public int DisposeCount { get; private set; } + + public OneShotEnumerable(object[] values) { this.values = values; } + + public IEnumerator GetEnumerator() + { + GetEnumeratorCount++; + if (GetEnumeratorCount > 1) + { + throw new InvalidOperationException("The enumerable was consumed more than once."); + } + return new Enumerator(this, values); + } + + private sealed class Enumerator : IEnumerator, IDisposable + { + private readonly OneShotEnumerable owner; + private readonly object[] values; + private int index = -1; + + public Enumerator(OneShotEnumerable owner, object[] values) + { + this.owner = owner; + this.values = values; + } + + public object Current { get { return values[index]; } } + public bool MoveNext() { index++; return index < values.Length; } + public void Reset() { throw new NotSupportedException(); } + public void Dispose() { owner.DisposeCount++; } + } + } +} +'@ + } + + function Get-YamlTestInfinitePipeline { + param ([string] $Item = 'value') + + while ($true) { + $script:YamlTestPipelineCount++ + $Item + } + } } Describe 'ConvertTo-Yaml' { @@ -348,6 +437,60 @@ Describe 'ConvertTo-Yaml' { { 'long' | ConvertTo-Yaml -MaxScalarLength 3 } | Should -Throw } + It 'stops infinite pipelines at the node budget' { + $script:YamlTestPipelineCount = 0 + + { Get-YamlTestInfinitePipeline | ConvertTo-Yaml -MaxNodes 4 } | + Should -Throw -ExpectedMessage '*configured limit of 4 nodes*' + $script:YamlTestPipelineCount | Should -BeLessOrEqual 4 + } + + It 'stops infinite pipelines at the first oversized scalar' { + $script:YamlTestPipelineCount = 0 + + { Get-YamlTestInfinitePipeline -Item 'long' | + ConvertTo-Yaml -MaxScalarLength 3 } | + Should -Throw -ExpectedMessage '*configured limit of 3 characters*' + $script:YamlTestPipelineCount | Should -Be 1 + } + + It 'stops and disposes infinite enumerables at the node budget' { + $source = [YamlTests.InfiniteEnumerable]::new() + + { ConvertTo-Yaml -InputObject $source -MaxNodes 4 } | + Should -Throw -ExpectedMessage '*configured limit of 4 nodes*' + $source.MoveNextCount | Should -Be 4 + $source.DisposeCount | Should -Be 1 + } + + It 'stops and disposes enumerables at the first oversized scalar' { + $source = [YamlTests.InfiniteEnumerable]::new('long') + + { ConvertTo-Yaml -InputObject $source -MaxScalarLength 3 } | + Should -Throw -ExpectedMessage '*configured limit of 3 characters*' + $source.MoveNextCount | Should -Be 1 + $source.DisposeCount | Should -Be 1 + } + + It 'does not re-enumerate repeated references' { + $source = [YamlTests.OneShotEnumerable]::new([object[]] @(1, 2)) + $inputObject = [ordered]@{ first = $source; second = $source } + + $yaml = ConvertTo-Yaml -InputObject $inputObject + + $source.GetEnumeratorCount | Should -Be 1 + $source.DisposeCount | Should -Be 1 + $yaml | Should -Match '&id001' + $yaml | Should -Match '\*id001' + } + + It 'prechecks Base64 output before allocating the encoded scalar' { + $bytes = [byte[]] @(1, 2, 3) + + { ConvertTo-Yaml -InputObject $bytes -MaxScalarLength 3 } | + Should -Throw -ExpectedMessage '*configured limit of 3 characters*' + } + It 'supports the public maximum depth and rejects the next level specifically' { $atLimit = [ordered]@{} $current = $atLimit From 810addef999f4f854b6ab5dcf29ee610f0dba116 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 23 Jul 2026 23:26:39 +0200 Subject: [PATCH 07/24] Emit explicit overlong mapping keys Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../private/ConvertTo-YamlFlowText.ps1 | 8 +++- .../Get-YamlEmissionImplicitKeyLength.ps1 | 20 ++++++++ src/functions/private/Write-YamlNodeText.ps1 | 10 +++- tests/ConvertTo-Yaml.Tests.ps1 | 47 +++++++++++++++++++ 4 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 src/functions/private/Get-YamlEmissionImplicitKeyLength.ps1 diff --git a/src/functions/private/ConvertTo-YamlFlowText.ps1 b/src/functions/private/ConvertTo-YamlFlowText.ps1 index d5c12e8..47060dd 100644 --- a/src/functions/private/ConvertTo-YamlFlowText.ps1 +++ b/src/functions/private/ConvertTo-YamlFlowText.ps1 @@ -132,7 +132,13 @@ function ConvertTo-YamlFlowText { continue } if ($frame.State -eq 'MappingValue') { - $frame.Parts.Add("$($frame.KeyText)`: $($frame.Child.Value)") + $keyNode = $frame.Node.Entries[$frame.Index].Key + $explicitKey = ( + Get-YamlEmissionImplicitKeyLength -Node $keyNode ` + -RenderedText $frame.KeyText + ) -gt 1024 + $keyPrefix = if ($explicitKey) { '? ' } else { '' } + $frame.Parts.Add("$keyPrefix$($frame.KeyText)`: $($frame.Child.Value)") $frame.Index++ $frame.State = 'MappingKey' } diff --git a/src/functions/private/Get-YamlEmissionImplicitKeyLength.ps1 b/src/functions/private/Get-YamlEmissionImplicitKeyLength.ps1 new file mode 100644 index 0000000..5a68c92 --- /dev/null +++ b/src/functions/private/Get-YamlEmissionImplicitKeyLength.ps1 @@ -0,0 +1,20 @@ +function Get-YamlEmissionImplicitKeyLength { + <# + .SYNOPSIS + Gets an emitted implicit key length in Unicode scalar values. + #> + [CmdletBinding()] + [OutputType([int])] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Node, + + [Parameter(Mandatory)] + [string] $RenderedText + ) + + if ($Node.Kind.Equals('Scalar', [System.StringComparison]::Ordinal)) { + return Get-YamlRuneCount -Text ([string] $Node.Value) + } + return Get-YamlRuneCount -Text $RenderedText +} diff --git a/src/functions/private/Write-YamlNodeText.ps1 b/src/functions/private/Write-YamlNodeText.ps1 index 5260afa..cc67624 100644 --- a/src/functions/private/Write-YamlNodeText.ps1 +++ b/src/functions/private/Write-YamlNodeText.ps1 @@ -45,12 +45,20 @@ function Write-YamlNodeText { if ($task.Type -eq 'Entry') { $keyText = ConvertTo-YamlFlowText -Node $task.Entry.Key ` -EmittedReferences $EmittedReferences + $explicitKey = ( + Get-YamlEmissionImplicitKeyLength -Node $task.Entry.Key ` + -RenderedText $keyText + ) -gt 1024 + if ($explicitKey) { + $spaces = ' ' * ($task.Level * $Indent) + [void] $Builder.Append($spaces).Append('? ').Append($keyText).Append("`n") + } $stack.Push([pscustomobject]@{ Type = 'Node' Node = $task.Entry.Value Entry = $null Level = $task.Level - LeadingText = "$keyText`: " + LeadingText = if ($explicitKey) { ': ' } else { "$keyText`: " } }) continue } diff --git a/tests/ConvertTo-Yaml.Tests.ps1 b/tests/ConvertTo-Yaml.Tests.ps1 index 607baa0..8048d49 100644 --- a/tests/ConvertTo-Yaml.Tests.ps1 +++ b/tests/ConvertTo-Yaml.Tests.ps1 @@ -261,6 +261,53 @@ Describe 'ConvertTo-Yaml' { $enumerator.Value | Should -Be 'value' } + It 'uses explicit keys when scalar keys exceed 1024 Unicode values' { + $atLimit = [System.Collections.Specialized.OrderedDictionary]::new() + $atLimit.Add(('x' * 1024), 'value') + $overLimit = [System.Collections.Specialized.OrderedDictionary]::new() + $overLimit.Add(('😀' * 1025), [ordered]@{ nested = 'value' }) + + $atLimitYaml = ConvertTo-Yaml -InputObject $atLimit + $overLimitYaml = ConvertTo-Yaml -InputObject $overLimit + $roundTrip = $overLimitYaml | ConvertFrom-Yaml -AsHashtable + + $atLimitYaml | Should -Not -Match '^\? ' + $overLimitYaml | Should -Match '^\? ' + $overLimitYaml | Should -Match '(?m)^: $' + ($overLimitYaml | Test-Yaml) | Should -BeTrue + $roundTrip.Contains(('😀' * 1025)) | Should -BeTrue + $roundTrip[('😀' * 1025)]['nested'] | Should -Be 'value' + } + + It 'uses explicit keys when rendered collection keys exceed 1024 values' { + $dictionary = [System.Collections.Specialized.OrderedDictionary]::new() + $key = [object[]] (0..299) + $dictionary.Add($key, 'value') + + $yaml = ConvertTo-Yaml -InputObject $dictionary + $roundTrip = $yaml | ConvertFrom-Yaml -AsHashtable + $enumerator = $roundTrip.GetEnumerator() + $null = $enumerator.MoveNext() + + $yaml | Should -Match '^\? \[' + ($yaml | Test-Yaml) | Should -BeTrue + $enumerator.Key | Should -Be $key + $enumerator.Value | Should -Be 'value' + } + + It 'uses explicit keys inside flow-rendered complex keys' { + $innerKey = [System.Collections.Specialized.OrderedDictionary]::new() + $innerKey.Add(('x' * 1025), 'inner') + $dictionary = [System.Collections.Specialized.OrderedDictionary]::new() + $dictionary.Add($innerKey, 'outer') + + $yaml = ConvertTo-Yaml -InputObject $dictionary + + $yaml | Should -Match '\{\? ' + ($yaml | Test-Yaml) | Should -BeTrue + { $yaml | ConvertFrom-Yaml -AsHashtable } | Should -Not -Throw + } + It 'rejects dictionary keys that normalize to the same YAML value' { $dictionary = [System.Collections.Specialized.OrderedDictionary]::new() $dictionary.Add([int] 1, 'int') From 3655dcc95d43da19bb2c83132cd9a9e8b00d0408 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 23 Jul 2026 23:28:39 +0200 Subject: [PATCH 08/24] Classify timestamp boundary failures Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../private/ConvertTo-YamlTimestampText.ps1 | 51 +++++++++++++++++++ .../private/Get-YamlSerializationShape.ps1 | 19 +------ tests/ConvertTo-Yaml.Tests.ps1 | 51 +++++++++++++++++++ 3 files changed, 104 insertions(+), 17 deletions(-) create mode 100644 src/functions/private/ConvertTo-YamlTimestampText.ps1 diff --git a/src/functions/private/ConvertTo-YamlTimestampText.ps1 b/src/functions/private/ConvertTo-YamlTimestampText.ps1 new file mode 100644 index 0000000..b6c76d4 --- /dev/null +++ b/src/functions/private/ConvertTo-YamlTimestampText.ps1 @@ -0,0 +1,51 @@ +function ConvertTo-YamlTimestampText { + <# + .SYNOPSIS + Formats a representable CLR timestamp for YAML emission. + #> + [CmdletBinding()] + [OutputType([string])] + param ( + [Parameter(Mandatory)] + [object] $Value + ) + + try { + if ($Value -is [datetimeoffset]) { + $offsetMinutes = $Value.Offset.TotalMinutes + if ($offsetMinutes -lt -840 -or $offsetMinutes -gt 840 -or + $offsetMinutes -ne [System.Math]::Truncate($offsetMinutes) -or + $Value.UtcTicks -lt [datetime]::MinValue.Ticks -or + $Value.UtcTicks -gt [datetime]::MaxValue.Ticks) { + throw [System.ArgumentOutOfRangeException]::new('Value') + } + return $Value.ToString( + 'o', + [System.Globalization.CultureInfo]::InvariantCulture + ) + } + + if ($Value.Kind -eq [System.DateTimeKind]::Local) { + return [datetimeoffset]::new($Value).ToString( + 'o', + [System.Globalization.CultureInfo]::InvariantCulture + ) + } + + $utc = if ($Value.Kind -eq [System.DateTimeKind]::Utc) { + $Value + } else { + [datetime]::SpecifyKind($Value, [System.DateTimeKind]::Utc) + } + return $utc.ToString( + "yyyy-MM-dd'T'HH:mm:ss.fffffff'Z'", + [System.Globalization.CultureInfo]::InvariantCulture + ) + } catch [System.ArgumentException] { + throw (New-YamlSerializationException -ErrorId 'YamlTimestampSerializationFailed' ` + -Message 'The timestamp cannot be represented with its local or explicit UTC offset.') + } catch [System.FormatException] { + throw (New-YamlSerializationException -ErrorId 'YamlTimestampSerializationFailed' ` + -Message 'The timestamp cannot be represented with its local or explicit UTC offset.') + } +} diff --git a/src/functions/private/Get-YamlSerializationShape.ps1 b/src/functions/private/Get-YamlSerializationShape.ps1 index 095ec4a..e85529f 100644 --- a/src/functions/private/Get-YamlSerializationShape.ps1 +++ b/src/functions/private/Get-YamlSerializationShape.ps1 @@ -162,29 +162,14 @@ function Get-YamlSerializationShape { } if (-not $isPropertyBag -and $Value -is [datetimeoffset]) { $scalar.Tag = 'tag:yaml.org,2002:timestamp' - $scalar.Value = $Value.ToString('o', [System.Globalization.CultureInfo]::InvariantCulture) + $scalar.Value = ConvertTo-YamlTimestampText -Value $Value $scalar.Style = 'DoubleQuoted' Confirm-YamlScalarLength -Node $scalar -State $State return [pscustomobject]@{ Kind = 'Scalar'; Node = $scalar; Values = $null } } if (-not $isPropertyBag -and $Value -is [datetime]) { $scalar.Tag = 'tag:yaml.org,2002:timestamp' - if ($Value.Kind -eq [System.DateTimeKind]::Local) { - $scalar.Value = ([datetimeoffset] $Value).ToString( - 'o', - [System.Globalization.CultureInfo]::InvariantCulture - ) - } else { - $utc = if ($Value.Kind -eq [System.DateTimeKind]::Utc) { - $Value - } else { - [datetime]::SpecifyKind($Value, [System.DateTimeKind]::Utc) - } - $scalar.Value = $utc.ToString( - "yyyy-MM-dd'T'HH:mm:ss.fffffff'Z'", - [System.Globalization.CultureInfo]::InvariantCulture - ) - } + $scalar.Value = ConvertTo-YamlTimestampText -Value $Value $scalar.Style = 'DoubleQuoted' Confirm-YamlScalarLength -Node $scalar -State $State return [pscustomobject]@{ Kind = 'Scalar'; Node = $scalar; Values = $null } diff --git a/tests/ConvertTo-Yaml.Tests.ps1 b/tests/ConvertTo-Yaml.Tests.ps1 index 8048d49..7e520ad 100644 --- a/tests/ConvertTo-Yaml.Tests.ps1 +++ b/tests/ConvertTo-Yaml.Tests.ps1 @@ -211,6 +211,57 @@ Describe 'ConvertTo-Yaml' { [Text.Encoding]::UTF8.GetString($result['binary']) | Should -Be 'hello' } + It 'serializes every representable DateTimeOffset boundary stably' { + $values = @( + [datetimeoffset]::MinValue + [datetimeoffset]::MaxValue + [datetimeoffset]::new( + [datetime]::new(1, 1, 1, 14, 0, 0), + [timespan]::FromHours(14) + ) + [datetimeoffset]::new( + [datetime]::new(9999, 12, 31, 9, 59, 59, 999).AddTicks(9999), + [timespan]::FromHours(-14) + ) + ) + + foreach ($value in $values) { + $yaml = ConvertTo-Yaml -InputObject $value + $roundTrip = $yaml | ConvertFrom-Yaml + + ($yaml | Test-Yaml) | Should -BeTrue + $roundTrip.UtcTicks | Should -Be $value.UtcTicks + } + } + + It 'classifies unrepresentable local DateTime boundaries' { + $errors = [System.Collections.Generic.List[object]]::new() + $successes = 0 + foreach ($value in @( + [datetime]::SpecifyKind([datetime]::MinValue, [DateTimeKind]::Local) + [datetime]::SpecifyKind([datetime]::MaxValue, [DateTimeKind]::Local) + )) { + try { + $yaml = ConvertTo-Yaml -InputObject $value + ($yaml | Test-Yaml) | Should -BeTrue + $successes++ + } catch { + $errors.Add($_) + } + } + + ($successes + $errors.Count) | Should -Be 2 + if ([TimeZoneInfo]::Local.BaseUtcOffset -ne [timespan]::Zero) { + $errors.Count | Should -BeGreaterThan 0 + } + foreach ($serializationError in $errors) { + $serializationError.FullyQualifiedErrorId | + Should -Be 'YamlTimestampSerializationFailed,ConvertTo-Yaml' + $serializationError.Exception.Message | + Should -Be 'The timestamp cannot be represented with its local or explicit UTC offset.' + } + } + It 'emits aliases for repeated byte arrays and preserves their identity' { $bytes = [Text.Encoding]::UTF8.GetBytes('hello') $inputObject = [ordered]@{ first = $bytes; second = $bytes } From b68b5b250a4c4c905a17ace6ede29c79f746908c Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 23 Jul 2026 23:36:41 +0200 Subject: [PATCH 09/24] Strengthen conformance value comparisons Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/Conformance.Tests.ps1 | 89 ++++++++++++++++- tests/tools/Invoke-YamlTestSuite.ps1 | 144 +++++++++++++++++++-------- 2 files changed, 190 insertions(+), 43 deletions(-) diff --git a/tests/Conformance.Tests.ps1 b/tests/Conformance.Tests.ps1 index 79a42f8..740d2a7 100644 --- a/tests/Conformance.Tests.ps1 +++ b/tests/Conformance.Tests.ps1 @@ -26,14 +26,18 @@ BeforeAll { throw "Expected one release archive root, but found $($suiteRoots.Count)." } $suiteDataPath = $suiteRoots[0].FullName + $runnerPath = Join-Path $PSScriptRoot 'tools\Invoke-YamlTestSuite.ps1' $suiteResults = @( - & (Join-Path $PSScriptRoot 'tools\Invoke-YamlTestSuite.ps1') ` + & $runnerPath ` -Path $suiteDataPath ` -CompareJson ` -CompareEvents ` -CompareOutYaml ` -CompareEmitRoundTrip ) + $emptySuitePath = Join-Path $TestDrive 'empty-suite' + $null = New-Item -Path $emptySuitePath -ItemType Directory + . $runnerPath -Path $emptySuitePath -CompareJson } Describe 'Released yaml-test-suite corpus accounting' { @@ -145,6 +149,89 @@ Describe 'Released yaml-test-suite corpus accounting' { Should -Be @('ConstructedValueMismatch') } + It 'canonicalizes binary, complex-key, ordered, and unsupported values distinctly' { + (ConvertTo-YamlSuiteCanonicalValue -Value ([byte[]] @(1, 2))) | + Should -Not -Be (ConvertTo-YamlSuiteCanonicalValue -Value ([byte[]] @(3, 4))) + + $firstComplex = [System.Collections.Specialized.OrderedDictionary]::new() + $secondComplex = [System.Collections.Specialized.OrderedDictionary]::new() + $firstComplex.Add([object[]] @('a'), 1) + $secondComplex.Add([object[]] @('b'), 1) + (ConvertTo-YamlSuiteCanonicalValue -Value $firstComplex) | + Should -Not -Be (ConvertTo-YamlSuiteCanonicalValue -Value $secondComplex) + + $firstOrder = [System.Collections.Specialized.OrderedDictionary]::new() + $secondOrder = [System.Collections.Specialized.OrderedDictionary]::new() + $firstOrder.Add('a', 1) + $firstOrder.Add('b', 2) + $secondOrder.Add('b', 2) + $secondOrder.Add('a', 1) + (ConvertTo-YamlSuiteCanonicalValue -Value $firstOrder) | + Should -Not -Be (ConvertTo-YamlSuiteCanonicalValue -Value $secondOrder) + + (ConvertTo-YamlSuiteCanonicalValue -Value ([uri] 'https://example.com/one')) | + Should -Not -Be ( + ConvertTo-YamlSuiteCanonicalValue -Value ([uri] 'https://example.com/two') + ) + } + + It 'includes binary and complex mapping-key identity in reference signatures' { + $sharedBytes = [byte[]] @(1, 2) + $sharedBinaryGraph = [object[]] @($sharedBytes, $sharedBytes) + $distinctBinaryGraph = [object[]] @([byte[]] @(1, 2), [byte[]] @(1, 2)) + (ConvertTo-YamlSuiteReferenceSignature -Value $sharedBinaryGraph) | + Should -Not -Be (ConvertTo-YamlSuiteReferenceSignature -Value $distinctBinaryGraph) + + $sharedKey = [object[]] @('key') + $sharedKeyGraph = [System.Collections.Specialized.OrderedDictionary]::new() + $sharedKeyGraph.Add($sharedKey, $sharedKey) + $distinctKeyGraph = [System.Collections.Specialized.OrderedDictionary]::new() + $distinctKeyGraph.Add([object[]] @('key'), [object[]] @('key')) + (ConvertTo-YamlSuiteReferenceSignature -Value $sharedKeyGraph) | + Should -Not -Be (ConvertTo-YamlSuiteReferenceSignature -Value $distinctKeyGraph) + } + + It 'detects altered ordered-map and alias semantics in out.yaml' { + $mutatedSuitePath = Join-Path $TestDrive 'mutated-out-cases' + $null = New-Item -Path $mutatedSuitePath -ItemType Directory -Force + foreach ($case in @('J7PZ', 'UGM3')) { + Copy-Item -LiteralPath (Join-Path $suiteDataPath $case) ` + -Destination $mutatedSuitePath -Recurse + } + + @' +--- !!omap +- Sammy Sosa: 63 +- Mark McGwire: 65 +- Ken Griffy: 58 +'@ | Set-Content -LiteralPath (Join-Path $mutatedSuitePath 'J7PZ\out.yaml') ` + -Encoding utf8NoBOM + + $invoicePath = Join-Path $mutatedSuitePath 'UGM3\out.yaml' + $invoice = Get-Content -LiteralPath $invoicePath -Raw + $duplicateAddress = @' +ship-to: + given: Chris + family: Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 + city: Royal Oak + state: MI + postal: 48046 +'@ + $invoice.Replace('ship-to: *id001', $duplicateAddress.TrimEnd()) | + Set-Content -LiteralPath $invoicePath -Encoding utf8NoBOM + + $mutatedResults = @( + & $runnerPath -Path $mutatedSuitePath -CompareOutYaml + ) + @($mutatedResults | Where-Object OutYamlResult -EQ 'Fail').Count | Should -Be 2 + @($mutatedResults | Sort-Object Case | Select-Object -ExpandProperty OutYamlReason) | + Should -Be @('OutYamlConstructionMismatch', 'OutYamlReferenceMismatch') + } + It 'accounts for out.yaml representation comparisons' { @($suiteResults | Where-Object OutYamlResult -EQ 'Pass').Count | Should -Be 241 @($suiteResults | Where-Object OutYamlResult -EQ 'PolicyDifference').Count | diff --git a/tests/tools/Invoke-YamlTestSuite.ps1 b/tests/tools/Invoke-YamlTestSuite.ps1 index 0fa559a..84c2253 100644 --- a/tests/tools/Invoke-YamlTestSuite.ps1 +++ b/tests/tools/Invoke-YamlTestSuite.ps1 @@ -115,7 +115,9 @@ function Split-YamlSuiteJsonDocument { function ConvertTo-YamlSuiteCanonicalValue { param ( [AllowNull()] - [object] $Value + [object] $Value, + + [switch] $SortMappings ) if ($null -eq $Value -or $Value -is [System.DBNull]) { @@ -127,6 +129,32 @@ function ConvertTo-YamlSuiteCanonicalValue { if ($Value -is [bool]) { return 'bool:{0}' -f $Value.ToString().ToLowerInvariant() } + if ($Value -is [byte[]]) { + return 'binary:{0}:{1}' -f $Value.Length, [System.Convert]::ToBase64String($Value) + } + if ($Value -is [char]) { + return 'char:{0}' -f [int] $Value + } + if ($Value.GetType().IsEnum) { + return 'enum:{0}:{1}' -f $Value.GetType().FullName, ( + [System.Convert]::ToUInt64($Value, [cultureinfo]::InvariantCulture) + ) + } + if ($Value -is [datetimeoffset]) { + return 'timestamp:{0}:{1}' -f $Value.UtcTicks, $Value.Offset.Ticks + } + if ($Value -is [datetime]) { + return 'datetime:{0}:{1}' -f $Value.Ticks, [int] $Value.Kind + } + if ($Value -is [timespan]) { + return 'timespan:{0}' -f $Value.Ticks + } + if ($Value -is [guid]) { + return 'guid:{0}' -f $Value.ToString('D') + } + if ($Value -is [uri]) { + return 'uri:{0}:{1}' -f $Value.OriginalString.Length, $Value.OriginalString + } $typeCode = [System.Type]::GetTypeCode($Value.GetType()) if ($Value -is [System.Numerics.BigInteger] -or $typeCode -in @( @@ -149,27 +177,48 @@ function ConvertTo-YamlSuiteCanonicalValue { } if ($Value -is [System.Collections.IDictionary]) { $entries = [System.Collections.Generic.List[string]]::new() - foreach ($key in $Value.Keys) { - if ($key -isnot [string]) { - return 'unsupported:non-string-mapping-key' - } - $canonicalValue = ConvertTo-YamlSuiteCanonicalValue -Value $Value[$key] - $entries.Add(('{0}:{1}={2}' -f $key.Length, $key, $canonicalValue)) + foreach ($entry in $Value.GetEnumerator()) { + $canonicalKey = ConvertTo-YamlSuiteCanonicalValue -Value $entry.Key ` + -SortMappings:$SortMappings + $canonicalValue = ConvertTo-YamlSuiteCanonicalValue -Value $entry.Value ` + -SortMappings:$SortMappings + $entries.Add(('{0}:{1}={2}:{3}' -f + $canonicalKey.Length, + $canonicalKey, + $canonicalValue.Length, + $canonicalValue + )) + } + $isOrdered = ( + $Value -is [System.Collections.Specialized.OrderedDictionary] -or + $Value.GetType().FullName -ceq 'System.Management.Automation.OrderedHashtable' + ) + if ($SortMappings -or -not $isOrdered) { + $entries.Sort([System.StringComparer]::Ordinal) } - $entries.Sort([System.StringComparer]::Ordinal) return 'map:{0}:{{{1}}}' -f $entries.Count, ($entries -join '|') } if ($Value -is [System.Collections.IEnumerable] -and $Value -isnot [string]) { - if ($Value -is [byte[]]) { - return 'unsupported:System.Byte[]' - } $items = [System.Collections.Generic.List[string]]::new() foreach ($item in $Value) { - $items.Add((ConvertTo-YamlSuiteCanonicalValue -Value $item)) + $items.Add(( + ConvertTo-YamlSuiteCanonicalValue -Value $item -SortMappings:$SortMappings + )) } return 'sequence:{0}:[{1}]' -f $items.Count, ($items -join '|') } - return 'unsupported:{0}' -f $Value.GetType().FullName + + $serialized = [System.Management.Automation.PSSerializer]::Serialize($Value, 3) + $payload = [System.Convert]::ToBase64String( + [System.Text.Encoding]::UTF8.GetBytes($serialized) + ) + if (-not $Value.GetType().IsValueType) { + return 'unsupported-reference:{0}:{1}:{2}' -f + $Value.GetType().FullName, + [System.Runtime.CompilerServices.RuntimeHelpers]::GetHashCode($Value), + $payload + } + return 'unsupported-value:{0}:{1}' -f $Value.GetType().FullName, $payload } function ConvertTo-YamlSuiteReferenceSignature { @@ -188,7 +237,7 @@ function ConvertTo-YamlSuiteReferenceSignature { while ($stack.Count -gt 0) { $frame = $stack.Pop() $current = $frame.Value - if ($null -eq $current -or $current -is [string] -or $current -is [byte[]]) { + if ($null -eq $current -or $current -is [string]) { continue } if ($current -isnot [System.Collections.IDictionary] -and @@ -206,13 +255,22 @@ function ConvertTo-YamlSuiteReferenceSignature { if (-not $first) { continue } + if ($current -is [byte[]]) { + continue + } if ($current -is [System.Collections.IDictionary]) { - foreach ($key in $current.Keys) { - $childPath = '{0}{{{1}}}' -f $frame.Path, (ConvertTo-YamlSuiteCanonicalValue -Value $key) + foreach ($entry in $current.GetEnumerator()) { + $childPath = '{0}{{{1}}}' -f $frame.Path, ( + ConvertTo-YamlSuiteCanonicalValue -Value $entry.Key + ) $stack.Push([pscustomobject]@{ - Value = $current[$key] - Path = $childPath + Value = $entry.Value + Path = "$childPath.value" + }) + $stack.Push([pscustomobject]@{ + Value = $entry.Key + Path = "$childPath.key" }) } continue @@ -333,9 +391,6 @@ function Test-YamlSuiteLegacyOrderedMapProjection { function Get-YamlSuiteJsonPolicyReason { [OutputType([string])] param ( - [Parameter(Mandatory)] - [string] $Case, - [Parameter(Mandatory)] [object[]] $ExpectedValues, @@ -343,22 +398,16 @@ function Get-YamlSuiteJsonPolicyReason { [object[]] $ActualValues ) - switch -CaseSensitive ($Case) { - '565N' { - if (Test-YamlSuiteBinaryByteArrayProjection -ExpectedValues $ExpectedValues ` - -ActualValues $ActualValues) { - return 'BinaryByteArrayProjection' - } - } - 'J7PZ' { - if (Test-YamlSuiteLegacyOrderedMapProjection -ExpectedValues $ExpectedValues ` - -ActualValues $ActualValues) { - return 'LegacyOrderedMapProjection' - } - } - default { return '' } + if (Test-YamlSuiteBinaryByteArrayProjection -ExpectedValues $ExpectedValues ` + -ActualValues $ActualValues) { + return 'BinaryByteArrayProjection' + } + if (Test-YamlSuiteLegacyOrderedMapProjection -ExpectedValues $ExpectedValues ` + -ActualValues $ActualValues) { + return 'LegacyOrderedMapProjection' } return '' + return '' } function ConvertFrom-YamlSuiteEventText { @@ -749,6 +798,7 @@ foreach ($inputFile in $inputFiles) { $stream = $null $projectedValues = $null $projectedCanonical = $null + $projectedJsonCanonical = $null $projectedReference = '' $projectionError = '' $eventExpected = $null @@ -756,6 +806,7 @@ foreach ($inputFile in $inputFiles) { $jsonExpected = $null $jsonActual = $null $outYamlCanonical = $null + $outYamlReference = $null $emitCanonical = $null $emitReference = $null @@ -801,6 +852,8 @@ foreach ($inputFile in $inputFiles) { try { $projectedValues = (Invoke-InYamlModule -ScriptBlock $projectYamlSuiteStream -Arguments (, $stream.Value)).Value $projectedCanonical = ConvertTo-YamlSuiteCanonicalValue -Value ([object[]] $projectedValues) + $projectedJsonCanonical = ConvertTo-YamlSuiteCanonicalValue ` + -Value ([object[]] $projectedValues) -SortMappings $projectedReference = ConvertTo-YamlSuiteReferenceSignature -Value ([object[]] $projectedValues) } catch { if ($_.Exception.Data.Contains('YamlErrorId')) { @@ -845,13 +898,14 @@ foreach ($inputFile in $inputFiles) { foreach ($document in $expectedDocuments) { $expectedValues.Add((ConvertFrom-Json -InputObject $document -AsHashtable -NoEnumerate)) } - $expectedCanonical = ConvertTo-YamlSuiteCanonicalValue -Value ([object[]] $expectedValues.ToArray()) + $expectedCanonical = ConvertTo-YamlSuiteCanonicalValue ` + -Value ([object[]] $expectedValues.ToArray()) -SortMappings $jsonExpected = $expectedCanonical - $jsonActual = $projectedCanonical - if ($projectedCanonical -ceq $expectedCanonical) { + $jsonActual = $projectedJsonCanonical + if ($projectedJsonCanonical -ceq $expectedCanonical) { $jsonResult = 'Pass' } else { - $reason = Get-YamlSuiteJsonPolicyReason -Case $casePath ` + $reason = Get-YamlSuiteJsonPolicyReason ` -ExpectedValues ([object[]] $expectedValues.ToArray()) ` -ActualValues ([object[]] $projectedValues) if ($reason) { @@ -877,12 +931,17 @@ foreach ($inputFile in $inputFiles) { $outStream = Invoke-InYamlModule -ScriptBlock $readYamlSuiteStream -Arguments @($outYaml) $outValues = (Invoke-InYamlModule -ScriptBlock $projectYamlSuiteStream -Arguments (, $outStream.Value)).Value $outCanonical = ConvertTo-YamlSuiteCanonicalValue -Value ([object[]] $outValues) + $outReference = ConvertTo-YamlSuiteReferenceSignature -Value ([object[]] $outValues) $outYamlCanonical = $outCanonical - if ($outCanonical -ceq $projectedCanonical) { - $outYamlResult = 'Pass' - } else { + $outYamlReference = $outReference + if ($outCanonical -cne $projectedCanonical) { $outYamlResult = 'Fail' $outYamlReason = 'OutYamlConstructionMismatch' + } elseif ($outReference -cne $projectedReference) { + $outYamlResult = 'Fail' + $outYamlReason = 'OutYamlReferenceMismatch' + } else { + $outYamlResult = 'Pass' } } catch { if ($_.Exception.Data.Contains('IsYamlException')) { @@ -975,6 +1034,7 @@ foreach ($inputFile in $inputFiles) { JsonExpected = $jsonExpected JsonActual = $jsonActual OutYamlActual = $outYamlCanonical + OutYamlRefs = $outYamlReference EmitActual = $emitCanonical EmitReferences = $emitReference ProjectedActual = $projectedCanonical From 992de6be7978604e5050be7e08e2f21031da7d1a Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 23 Jul 2026 23:41:53 +0200 Subject: [PATCH 10/24] Separate YAML emission conformance surfaces Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 14 +- tests/Conformance.Tests.ps1 | 57 +++++-- tests/tools/Invoke-YamlTestSuite.ps1 | 221 +++++++++++++++++++-------- 3 files changed, 217 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 76f1097..e0d8502 100644 --- a/README.md +++ b/README.md @@ -175,20 +175,24 @@ The archive contains 402 inputs: | Syntax and composition | 400 | 2 | 0 | 0 | | Representation events | 308 | 0 | 0 | 94 | | JSON projection | 277 | 2 | 0 | 123 | -| `out.yaml` projection | 241 | 0 | 0 | 161 | -| Emit and round trip | 306 | 0 | 0 | 96 | +| `out.yaml` projection | 241 | 1 | 0 | 160 | +| Official `emit.yaml` fixtures | 55 | 0 | 0 | 347 | +| Module self-round-trip | 306 | 2 | 0 | 94 | All 94 fixtures marked invalid are rejected. The valid `2JQS` and `X38W` inputs are syntactically recognized and produce matching representation events, then are rejected during load validation because representation -mapping keys must be unique. They are not unsupported grammar. +mapping keys must be unique. They are not unsupported grammar. Both are +reported as policy differences for module self-round-trip; `X38W`, the one +case with an `out.yaml` fixture, is also reported that way on that surface. The two JSON projection differences are `565N`, where `!!binary` intentionally becomes `byte[]` instead of a Base64 string, and `J7PZ`, where legacy `!!omap` intentionally becomes `System.Collections.Specialized.OrderedDictionary` instead of remaining a sequence of one-entry mappings. No event mismatch is -classified as policy. The deterministic runner reports no unexplained -failures. +classified as policy. All 55 official `emit.yaml` fixtures are read and +validated independently of the 402-input module self-round-trip. The +deterministic runner reports no unexplained failures. These results are a pinned compatibility measurement, not a claim that a finite corpus proves complete YAML 1.2.2 compliance. diff --git a/tests/Conformance.Tests.ps1 b/tests/Conformance.Tests.ps1 index 740d2a7..15a028b 100644 --- a/tests/Conformance.Tests.ps1 +++ b/tests/Conformance.Tests.ps1 @@ -33,7 +33,8 @@ BeforeAll { -CompareJson ` -CompareEvents ` -CompareOutYaml ` - -CompareEmitRoundTrip + -CompareEmitYaml ` + -CompareSelfRoundTrip ) $emptySuitePath = Join-Path $TestDrive 'empty-suite' $null = New-Item -Path $emptySuitePath -ItemType Directory @@ -235,19 +236,57 @@ ship-to: It 'accounts for out.yaml representation comparisons' { @($suiteResults | Where-Object OutYamlResult -EQ 'Pass').Count | Should -Be 241 @($suiteResults | Where-Object OutYamlResult -EQ 'PolicyDifference').Count | - Should -Be 0 + Should -Be 1 @($suiteResults | Where-Object OutYamlResult -EQ 'Fail').Count | Should -Be 0 @($suiteResults | Where-Object OutYamlResult -EQ 'NotApplicable').Count | - Should -Be 161 + Should -Be 160 + $outPolicy = $suiteResults | Where-Object OutYamlResult -EQ 'PolicyDifference' + $outPolicy.Case | Should -Be 'X38W' + $outPolicy.OutYamlReason | Should -Be 'RepresentationMappingKeyUniqueness' + } + + It 'validates the 55 official emit.yaml fixtures separately' { + @($suiteResults | Where-Object HasEmitYaml).Count | Should -Be 55 + @($suiteResults | Where-Object EmitYamlResult -EQ 'Pass').Count | Should -Be 55 + @($suiteResults | Where-Object EmitYamlResult -EQ 'PolicyDifference').Count | + Should -Be 0 + @($suiteResults | Where-Object EmitYamlResult -EQ 'Fail').Count | Should -Be 0 + @($suiteResults | Where-Object EmitYamlResult -EQ 'NotApplicable').Count | + Should -Be 347 + } + + It 'reads official emit.yaml content rather than counting its presence' { + $mutatedSuitePath = Join-Path $TestDrive 'mutated-emit-fixture' + $null = New-Item -Path $mutatedSuitePath -ItemType Directory -Force + Copy-Item -LiteralPath (Join-Path $suiteDataPath '2LFX') ` + -Destination $mutatedSuitePath -Recurse + '--- altered' | Set-Content ` + -LiteralPath (Join-Path $mutatedSuitePath '2LFX\emit.yaml') ` + -Encoding utf8NoBOM + + $mutatedResult = & $runnerPath -Path $mutatedSuitePath -CompareEmitYaml + + $mutatedResult.EmitYamlResult | Should -Be 'Fail' + $mutatedResult.EmitYamlReason | Should -Be 'EmitYamlRepresentationMismatch' } - It 'accounts for emitter and round-trip comparisons' { - @($suiteResults | Where-Object EmitResult -EQ 'Pass').Count | Should -Be 306 - @($suiteResults | Where-Object EmitResult -EQ 'PolicyDifference').Count | + It 'accounts honestly for general module self-round-trips' { + @($suiteResults | Where-Object SelfRoundTripResult -EQ 'Pass').Count | + Should -Be 306 + @($suiteResults | Where-Object SelfRoundTripResult -EQ 'PolicyDifference').Count | + Should -Be 2 + @($suiteResults | Where-Object SelfRoundTripResult -EQ 'Fail').Count | Should -Be 0 - @($suiteResults | Where-Object EmitResult -EQ 'Fail').Count | Should -Be 0 - @($suiteResults | Where-Object EmitResult -EQ 'NotApplicable').Count | - Should -Be 96 + @($suiteResults | Where-Object SelfRoundTripResult -EQ 'NotApplicable').Count | + Should -Be 94 + $policyResults = @( + $suiteResults | + Where-Object SelfRoundTripResult -EQ 'PolicyDifference' | + Sort-Object Case + ) + @($policyResults.Case) | Should -Be @('2JQS', 'X38W') + @($policyResults.SelfRoundTripReason | Select-Object -Unique) | + Should -Be @('RepresentationMappingKeyUniqueness') } It 'keeps the previously failing multi-document JSON cases green' { diff --git a/tests/tools/Invoke-YamlTestSuite.ps1 b/tests/tools/Invoke-YamlTestSuite.ps1 index 84c2253..f7294d9 100644 --- a/tests/tools/Invoke-YamlTestSuite.ps1 +++ b/tests/tools/Invoke-YamlTestSuite.ps1 @@ -11,7 +11,8 @@ param ( [switch] $CompareJson, [switch] $CompareEvents, [switch] $CompareOutYaml, - [switch] $CompareEmitRoundTrip + [switch] $CompareEmitYaml, + [switch] $CompareSelfRoundTrip ) . (Join-Path $PSScriptRoot '..\TestBootstrap.ps1') @@ -19,11 +20,13 @@ param ( if (-not $PSBoundParameters.ContainsKey('CompareJson') -and -not $PSBoundParameters.ContainsKey('CompareEvents') -and -not $PSBoundParameters.ContainsKey('CompareOutYaml') -and - -not $PSBoundParameters.ContainsKey('CompareEmitRoundTrip')) { + -not $PSBoundParameters.ContainsKey('CompareEmitYaml') -and + -not $PSBoundParameters.ContainsKey('CompareSelfRoundTrip')) { $CompareJson = $true $CompareEvents = $true $CompareOutYaml = $true - $CompareEmitRoundTrip = $true + $CompareEmitYaml = $true + $CompareSelfRoundTrip = $true } function Invoke-InYamlModule { @@ -754,6 +757,12 @@ $projectYamlSuiteText = { } New-YamlValueBox -Value ([object[]] $values.ToArray()) } +$testYamlSuiteText = { + param ([string] $YamlText) + Test-Yaml -Yaml $YamlText -Depth 128 -MaxNodes 100000 -MaxAliases 1000 ` + -MaxScalarLength 1048576 -MaxTagLength 1024 -MaxTotalTagLength 65536 ` + -MaxNumericLength 4096 +} $suiteRoot = (Resolve-Path -LiteralPath $Path).Path $inputFiles = @( @@ -791,8 +800,10 @@ foreach ($inputFile in $inputFiles) { $jsonReason = '' $outYamlResult = 'NotApplicable' $outYamlReason = '' - $emitResult = 'NotApplicable' - $emitReason = '' + $emitYamlResult = 'NotApplicable' + $emitYamlReason = '' + $selfRoundTripResult = 'NotApplicable' + $selfRoundTripReason = '' $representation = $null $stream = $null @@ -807,8 +818,12 @@ foreach ($inputFile in $inputFiles) { $jsonActual = $null $outYamlCanonical = $null $outYamlReference = $null - $emitCanonical = $null - $emitReference = $null + $emitYamlExpected = $null + $emitYamlActual = $null + $emitYamlExpectedReference = $null + $emitYamlActualReference = $null + $selfRoundTripCanonical = $null + $selfRoundTripReference = $null try { $representation = Invoke-InYamlModule -ScriptBlock $readYamlSuiteRepresentation -Arguments @($yaml) @@ -920,10 +935,12 @@ foreach ($inputFile in $inputFiles) { } if ($CompareOutYaml -and $hasOutYaml) { - if ($null -eq $stream -or $expectsError -or $syntaxResult -eq 'PolicyDifference' -or + if ($syntaxResult -eq 'PolicyDifference') { + $outYamlResult = 'PolicyDifference' + $outYamlReason = $syntaxReason + } elseif ($null -eq $stream -or $expectsError -or $null -eq $projectedValues) { $outYamlResult = 'NotApplicable' - if ($syntaxResult -eq 'PolicyDifference') { $outYamlReason = $syntaxReason } if ($projectionError) { $outYamlReason = $projectionError } } else { $outYaml = [System.IO.File]::ReadAllText($outYamlPath, [System.Text.UTF8Encoding]::new($false, $true)) @@ -954,16 +971,96 @@ foreach ($inputFile in $inputFiles) { } } - if ($CompareEmitRoundTrip) { - if ($null -eq $stream -or $expectsError) { - $emitResult = 'NotApplicable' - if ($expectsError) { $emitReason = 'InvalidSyntax' } + if ($CompareEmitYaml -and $hasEmitYaml) { + try { + $emitYaml = [System.IO.File]::ReadAllText( + $emitYamlPath, + [System.Text.UTF8Encoding]::new($false, $true) + ) + $isValidFixture = Invoke-InYamlModule -ScriptBlock $testYamlSuiteText ` + -Arguments @($emitYaml) + if (-not $isValidFixture) { + $emitYamlResult = 'Fail' + $emitYamlReason = 'EmitYamlInvalid' + } else { + $fixtureValues = (Invoke-InYamlModule -ScriptBlock $projectYamlSuiteText ` + -Arguments @($emitYaml)).Value + $fixtureCanonical = ConvertTo-YamlSuiteCanonicalValue ` + -Value ([object[]] $fixtureValues) + $fixtureReference = ConvertTo-YamlSuiteReferenceSignature ` + -Value ([object[]] $fixtureValues) + + if ($null -ne $projectedCanonical -and + ($fixtureCanonical -cne $projectedCanonical -or + $fixtureReference -cne $projectedReference)) { + $emitYamlResult = 'Fail' + $emitYamlReason = 'EmitYamlRepresentationMismatch' + $emitYamlExpected = $projectedCanonical + $emitYamlActual = $fixtureCanonical + $emitYamlExpectedReference = $projectedReference + $emitYamlActualReference = $fixtureReference + } else { + $fixtureEmittedDocuments = [System.Collections.Generic.List[string]]::new() + foreach ($fixtureValue in $fixtureValues) { + $fixtureEmitted = Invoke-InYamlModule -ScriptBlock { + param ($InputValue) + ConvertTo-Yaml -InputObject $InputValue -ExplicitDocumentStart + } -Arguments (, $fixtureValue) + $fixtureEmittedDocuments.Add([string] $fixtureEmitted) + } + $fixtureEmittedText = $fixtureEmittedDocuments.ToArray() -join "`n" + $isValidFixtureEmit = Invoke-InYamlModule ` + -ScriptBlock $testYamlSuiteText -Arguments @($fixtureEmittedText) + if (-not $isValidFixtureEmit) { + $emitYamlResult = 'Fail' + $emitYamlReason = 'EmittedYamlInvalid' + } else { + $fixtureRoundTripValues = ( + Invoke-InYamlModule -ScriptBlock $projectYamlSuiteText ` + -Arguments @($fixtureEmittedText) + ).Value + $fixtureRoundCanonical = ConvertTo-YamlSuiteCanonicalValue ` + -Value ([object[]] $fixtureRoundTripValues) + $fixtureRoundReference = ConvertTo-YamlSuiteReferenceSignature ` + -Value ([object[]] $fixtureRoundTripValues) + $emitYamlExpected = $fixtureCanonical + $emitYamlActual = $fixtureRoundCanonical + $emitYamlExpectedReference = $fixtureReference + $emitYamlActualReference = $fixtureRoundReference + } + if ($emitYamlResult -ne 'Fail' -and + $fixtureRoundCanonical -ceq $fixtureCanonical -and + $fixtureRoundReference -ceq $fixtureReference) { + $emitYamlResult = 'Pass' + } elseif ($emitYamlResult -ne 'Fail') { + $emitYamlResult = 'Fail' + $emitYamlReason = 'EmitYamlRoundTripMismatch' + } + } + } + } catch [System.NotSupportedException] { + $emitYamlResult = 'Fail' + $emitYamlReason = 'UnsupportedEmissionType' + } catch { + if ($_.Exception.Data.Contains('IsYamlException')) { + $emitYamlResult = 'Fail' + $emitYamlReason = [string] $_.Exception.Data['YamlErrorId'] + } else { + throw + } + } + } + + if ($CompareSelfRoundTrip) { + if ($syntaxResult -eq 'PolicyDifference') { + $selfRoundTripResult = 'PolicyDifference' + $selfRoundTripReason = $syntaxReason + } elseif ($null -eq $stream -or $expectsError) { + $selfRoundTripResult = 'NotApplicable' + if ($expectsError) { $selfRoundTripReason = 'InvalidSyntax' } } elseif ($projectionError) { - $emitResult = 'Fail' - $emitReason = $projectionError - } elseif ($syntaxResult -eq 'PolicyDifference') { - $emitResult = 'PolicyDifference' - $emitReason = $syntaxReason + $selfRoundTripResult = 'Fail' + $selfRoundTripReason = $projectionError } else { try { $emittedDocuments = [System.Collections.Generic.List[string]]::new() @@ -975,36 +1072,32 @@ foreach ($inputFile in $inputFiles) { $emittedDocuments.Add([string] $emitted) } $emittedText = ($emittedDocuments.ToArray() -join "`n") - $isValidEmit = Invoke-InYamlModule -ScriptBlock { - param ($YamlText) - Test-Yaml -Yaml $YamlText -Depth 128 -MaxNodes 100000 -MaxAliases 1000 ` - -MaxScalarLength 1048576 -MaxTagLength 1024 -MaxTotalTagLength 65536 ` - -MaxNumericLength 4096 - } -Arguments @($emittedText) + $isValidEmit = Invoke-InYamlModule -ScriptBlock $testYamlSuiteText ` + -Arguments @($emittedText) if (-not $isValidEmit) { - $emitResult = 'Fail' - $emitReason = 'EmittedYamlInvalid' + $selfRoundTripResult = 'Fail' + $selfRoundTripReason = 'EmittedYamlInvalid' } else { $roundTripValues = (Invoke-InYamlModule -ScriptBlock $projectYamlSuiteText ` -Arguments @($emittedText)).Value $roundCanonical = ConvertTo-YamlSuiteCanonicalValue -Value ([object[]] $roundTripValues) $roundReference = ConvertTo-YamlSuiteReferenceSignature -Value ([object[]] $roundTripValues) - $emitCanonical = $roundCanonical - $emitReference = $roundReference + $selfRoundTripCanonical = $roundCanonical + $selfRoundTripReference = $roundReference if ($roundCanonical -ceq $projectedCanonical -and $roundReference -ceq $projectedReference) { - $emitResult = 'Pass' + $selfRoundTripResult = 'Pass' } else { - $emitResult = 'Fail' - $emitReason = 'EmitRoundTripMismatch' + $selfRoundTripResult = 'Fail' + $selfRoundTripReason = 'SelfRoundTripMismatch' } } } catch [System.NotSupportedException] { - $emitResult = 'Fail' - $emitReason = 'UnsupportedEmissionType' + $selfRoundTripResult = 'Fail' + $selfRoundTripReason = 'UnsupportedEmissionType' } catch { if ($_.Exception.Data.Contains('IsYamlException')) { - $emitResult = 'Fail' - $emitReason = [string] $_.Exception.Data['YamlErrorId'] + $selfRoundTripResult = 'Fail' + $selfRoundTripReason = [string] $_.Exception.Data['YamlErrorId'] } else { throw } @@ -1013,31 +1106,37 @@ foreach ($inputFile in $inputFiles) { } [pscustomobject]@{ - Case = $casePath - ExpectsError = $expectsError - HasJson = $hasJson - HasEvent = $hasEvent - HasOutYaml = $hasOutYaml - HasEmitYaml = $hasEmitYaml - SyntaxResult = $syntaxResult - SyntaxReason = $syntaxReason - EventResult = $eventResult - EventReason = $eventReason - JsonResult = $jsonResult - JsonReason = $jsonReason - OutYamlResult = $outYamlResult - OutYamlReason = $outYamlReason - EmitResult = $emitResult - EmitReason = $emitReason - EventExpected = $eventExpected - EventActual = $eventActual - JsonExpected = $jsonExpected - JsonActual = $jsonActual - OutYamlActual = $outYamlCanonical - OutYamlRefs = $outYamlReference - EmitActual = $emitCanonical - EmitReferences = $emitReference - ProjectedActual = $projectedCanonical - ProjectedRefs = $projectedReference + Case = $casePath + ExpectsError = $expectsError + HasJson = $hasJson + HasEvent = $hasEvent + HasOutYaml = $hasOutYaml + HasEmitYaml = $hasEmitYaml + SyntaxResult = $syntaxResult + SyntaxReason = $syntaxReason + EventResult = $eventResult + EventReason = $eventReason + JsonResult = $jsonResult + JsonReason = $jsonReason + OutYamlResult = $outYamlResult + OutYamlReason = $outYamlReason + EmitYamlResult = $emitYamlResult + EmitYamlReason = $emitYamlReason + SelfRoundTripResult = $selfRoundTripResult + SelfRoundTripReason = $selfRoundTripReason + EventExpected = $eventExpected + EventActual = $eventActual + JsonExpected = $jsonExpected + JsonActual = $jsonActual + OutYamlActual = $outYamlCanonical + OutYamlRefs = $outYamlReference + EmitYamlExpected = $emitYamlExpected + EmitYamlActual = $emitYamlActual + EmitYamlExpectedRefs = $emitYamlExpectedReference + EmitYamlActualRefs = $emitYamlActualReference + SelfRoundTripActual = $selfRoundTripCanonical + SelfRoundTripRefs = $selfRoundTripReference + ProjectedActual = $projectedCanonical + ProjectedRefs = $projectedReference } } From 9b34e90638c574b1fe714cd93f495dcefbdb490c Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 23 Jul 2026 23:45:19 +0200 Subject: [PATCH 11/24] Keep serialization regressions analyzer-clean Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/ConvertTo-Yaml.Tests.ps1 | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/ConvertTo-Yaml.Tests.ps1 b/tests/ConvertTo-Yaml.Tests.ps1 index 7e520ad..19f8387 100644 --- a/tests/ConvertTo-Yaml.Tests.ps1 +++ b/tests/ConvertTo-Yaml.Tests.ps1 @@ -95,6 +95,10 @@ namespace YamlTests } function Get-YamlTestInfinitePipeline { + <# + .SYNOPSIS + Produces an unbounded stream for resource-limit tests. + #> param ([string] $Item = 'value') while ($true) { @@ -316,7 +320,8 @@ Describe 'ConvertTo-Yaml' { $atLimit = [System.Collections.Specialized.OrderedDictionary]::new() $atLimit.Add(('x' * 1024), 'value') $overLimit = [System.Collections.Specialized.OrderedDictionary]::new() - $overLimit.Add(('😀' * 1025), [ordered]@{ nested = 'value' }) + $overLimitKey = [char]::ConvertFromUtf32(0x1F600) * 1025 + $overLimit.Add($overLimitKey, [ordered]@{ nested = 'value' }) $atLimitYaml = ConvertTo-Yaml -InputObject $atLimit $overLimitYaml = ConvertTo-Yaml -InputObject $overLimit @@ -326,8 +331,8 @@ Describe 'ConvertTo-Yaml' { $overLimitYaml | Should -Match '^\? ' $overLimitYaml | Should -Match '(?m)^: $' ($overLimitYaml | Test-Yaml) | Should -BeTrue - $roundTrip.Contains(('😀' * 1025)) | Should -BeTrue - $roundTrip[('😀' * 1025)]['nested'] | Should -Be 'value' + $roundTrip.Contains($overLimitKey) | Should -BeTrue + $roundTrip[$overLimitKey]['nested'] | Should -Be 'value' } It 'uses explicit keys when rendered collection keys exceed 1024 values' { @@ -546,9 +551,11 @@ Describe 'ConvertTo-Yaml' { It 'stops infinite pipelines at the first oversized scalar' { $script:YamlTestPipelineCount = 0 - { Get-YamlTestInfinitePipeline -Item 'long' | - ConvertTo-Yaml -MaxScalarLength 3 } | - Should -Throw -ExpectedMessage '*configured limit of 3 characters*' + $conversion = { + Get-YamlTestInfinitePipeline -Item 'long' | + ConvertTo-Yaml -MaxScalarLength 3 + } + $conversion | Should -Throw -ExpectedMessage '*configured limit of 3 characters*' $script:YamlTestPipelineCount | Should -Be 1 } From 48ff4f312c0887a642518174d0bbc0ea716f3ea7 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 24 Jul 2026 00:16:38 +0200 Subject: [PATCH 12/24] Close YAML parser boundary gaps Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../private/ConvertFrom-YamlByteOrderMark.ps1 | 19 +++------ .../private/Get-YamlEffectiveTag.ps1 | 6 ++- .../Get-YamlEmissionImplicitKeyLength.ps1 | 3 -- .../private/Get-YamlImplicitKeyLength.ps1 | 4 -- .../private/Read-YamlBlockMapping.ps1 | 2 +- .../private/Test-YamlDocumentPrefix.ps1 | 42 +++++++++++++++++++ tests/ConvertFrom-Yaml.Tests.ps1 | 3 +- tests/ConvertTo-Yaml.Tests.ps1 | 4 +- tests/Test-Yaml.Tests.ps1 | 31 +++++++++----- 9 files changed, 78 insertions(+), 36 deletions(-) create mode 100644 src/functions/private/Test-YamlDocumentPrefix.ps1 diff --git a/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 b/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 index ae12d96..8ede89b 100644 --- a/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 +++ b/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 @@ -24,20 +24,11 @@ function ConvertFrom-YamlByteOrderMark { $atStreamStart = $index -eq 0 $atLineStart = $index -gt 0 -and $Text[$index - 1] -ceq "`n" - $beforeDirective = $index + 1 -lt $Text.Length -and $Text[$index + 1] -ceq '%' - $beforeDocumentStart = $false - if ($index + 3 -lt $Text.Length -and - $Text.Substring($index + 1, 3).Equals( - '---', - [System.StringComparison]::Ordinal - )) { - $afterMarker = $index + 4 - $beforeDocumentStart = $afterMarker -ge $Text.Length -or - (Test-YamlWhiteSpace -Character $Text[$afterMarker]) -or - $Text[$afterMarker] -ceq "`n" - } + $beforeDocumentPrefix = $atLineStart -and ( + Test-YamlDocumentPrefix -Text $Text -Index ($index + 1) + ) - if ($atStreamStart -or ($atLineStart -and ($beforeDirective -or $beforeDocumentStart))) { + if ($atStreamStart -or $beforeDocumentPrefix) { continue } @@ -47,7 +38,7 @@ function ConvertFrom-YamlByteOrderMark { $column = if ($lastBreak -lt 0) { $before.Length } else { $before.Length - $lastBreak - 1 } $mark = New-YamlMark -Index $index -Line $line -Column $column throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidByteOrderMark' -Message ( - 'A YAML byte order mark is only allowed at the start of the stream or an explicit document.' + 'A YAML byte order mark is only allowed at the start of the stream or a document prefix.' )) } diff --git a/src/functions/private/Get-YamlEffectiveTag.ps1 b/src/functions/private/Get-YamlEffectiveTag.ps1 index ea70947..5619941 100644 --- a/src/functions/private/Get-YamlEffectiveTag.ps1 +++ b/src/functions/private/Get-YamlEffectiveTag.ps1 @@ -14,7 +14,8 @@ function Get-YamlEffectiveTag { [object] $Value ) - if (-not [string]::IsNullOrEmpty($Node.Tag)) { + if (-not [string]::IsNullOrEmpty($Node.Tag) -and + -not $Node.Tag.Equals('!', [System.StringComparison]::Ordinal)) { return $Node.Tag } if ($Node.Kind.Equals('Sequence', [System.StringComparison]::Ordinal)) { @@ -23,6 +24,9 @@ function Get-YamlEffectiveTag { if ($Node.Kind.Equals('Mapping', [System.StringComparison]::Ordinal)) { return 'tag:yaml.org,2002:map' } + if ($Node.Tag.Equals('!', [System.StringComparison]::Ordinal)) { + return 'tag:yaml.org,2002:str' + } if ($null -eq $Value) { return 'tag:yaml.org,2002:null' diff --git a/src/functions/private/Get-YamlEmissionImplicitKeyLength.ps1 b/src/functions/private/Get-YamlEmissionImplicitKeyLength.ps1 index 5a68c92..4962d36 100644 --- a/src/functions/private/Get-YamlEmissionImplicitKeyLength.ps1 +++ b/src/functions/private/Get-YamlEmissionImplicitKeyLength.ps1 @@ -13,8 +13,5 @@ function Get-YamlEmissionImplicitKeyLength { [string] $RenderedText ) - if ($Node.Kind.Equals('Scalar', [System.StringComparison]::Ordinal)) { - return Get-YamlRuneCount -Text ([string] $Node.Value) - } return Get-YamlRuneCount -Text $RenderedText } diff --git a/src/functions/private/Get-YamlImplicitKeyLength.ps1 b/src/functions/private/Get-YamlImplicitKeyLength.ps1 index ad583b4..63bee94 100644 --- a/src/functions/private/Get-YamlImplicitKeyLength.ps1 +++ b/src/functions/private/Get-YamlImplicitKeyLength.ps1 @@ -13,10 +13,6 @@ function Get-YamlImplicitKeyLength { [pscustomobject] $Context ) - if ($Node.Kind.Equals('Scalar', [System.StringComparison]::Ordinal)) { - return Get-YamlRuneCount -Text ([string] $Node.Value) - } - $sourceLength = [Math]::Max(0, $Node.End.Index - $Node.Start.Index) return Get-YamlRuneCount -Text $Context.Text.Substring($Node.Start.Index, $sourceLength) } diff --git a/src/functions/private/Read-YamlBlockMapping.ps1 b/src/functions/private/Read-YamlBlockMapping.ps1 index 6eb3d25..ab8c0d1 100644 --- a/src/functions/private/Read-YamlBlockMapping.ps1 +++ b/src/functions/private/Read-YamlBlockMapping.ps1 @@ -191,7 +191,7 @@ function Read-YamlBlockMapping { $key = Read-YamlBlockKey -Context $Context -Text $keyText -Line $lineNumber ` -Column $contentColumn -Depth ($Depth + 1) } - if ((Get-YamlImplicitKeyLength -Node $key -Context $Context) -gt 1024) { + if ($colon -gt 0 -and (Get-YamlRuneCount -Text $keyText) -gt 1024) { $mark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $contentColumn + $colon) ` -Line $lineNumber -Column ($contentColumn + $colon) throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidImplicitKey' -Message ( diff --git a/src/functions/private/Test-YamlDocumentPrefix.ps1 b/src/functions/private/Test-YamlDocumentPrefix.ps1 new file mode 100644 index 0000000..d15adf5 --- /dev/null +++ b/src/functions/private/Test-YamlDocumentPrefix.ps1 @@ -0,0 +1,42 @@ +function Test-YamlDocumentPrefix { + <# + .SYNOPSIS + Tests whether text after a BOM is a legal YAML document prefix. + #> + [CmdletBinding()] + [OutputType([bool])] + param ( + [Parameter(Mandatory)] + [string] $Text, + + [Parameter(Mandatory)] + [ValidateRange(0, 2147483647)] + [int] $Index + ) + + while ($Index -lt $Text.Length) { + $lineEnd = $Text.IndexOf("`n", $Index, [System.StringComparison]::Ordinal) + if ($lineEnd -lt 0) { + $lineEnd = $Text.Length + } + $line = $Text.Substring($Index, $lineEnd - $Index) + $trimmed = $line.TrimStart(' ', "`t") + if ($trimmed.Length -eq 0 -or + $trimmed.StartsWith('#', [System.StringComparison]::Ordinal)) { + if ($lineEnd -ge $Text.Length) { + return $false + } + $Index = $lineEnd + 1 + continue + } + if ($line.StartsWith('%', [System.StringComparison]::Ordinal)) { + return $true + } + if (-not $line.StartsWith('---', [System.StringComparison]::Ordinal)) { + return $false + } + return $line.Length -eq 3 -or + (Test-YamlWhiteSpace -Character $line[3]) + } + return $false +} diff --git a/tests/ConvertFrom-Yaml.Tests.ps1 b/tests/ConvertFrom-Yaml.Tests.ps1 index 8880b37..7a69e3d 100644 --- a/tests/ConvertFrom-Yaml.Tests.ps1 +++ b/tests/ConvertFrom-Yaml.Tests.ps1 @@ -204,7 +204,7 @@ folded: > It 'consumes byte order marks only at legal document boundaries' { $bom = [char] 0xFEFF $documents = @( - "${bom}%YAML 1.2`n---`none`n...`n${bom}---`ntwo" | + "${bom}%YAML 1.2`n---`none`n...`n${bom}# prefix comment`n`n---`ntwo" | ConvertFrom-Yaml ) @@ -212,6 +212,7 @@ folded: > ("${bom}---`nvalue" | ConvertFrom-Yaml) | Should -Be 'value' ("foo${bom}bar" | Test-Yaml) | Should -BeFalse ("---`n${bom}value" | Test-Yaml) | Should -BeFalse + ("---`none`n...`n${bom}# comment`ntwo" | Test-Yaml) | Should -BeFalse } } diff --git a/tests/ConvertTo-Yaml.Tests.ps1 b/tests/ConvertTo-Yaml.Tests.ps1 index 19f8387..3f0de95 100644 --- a/tests/ConvertTo-Yaml.Tests.ps1 +++ b/tests/ConvertTo-Yaml.Tests.ps1 @@ -318,9 +318,9 @@ Describe 'ConvertTo-Yaml' { It 'uses explicit keys when scalar keys exceed 1024 Unicode values' { $atLimit = [System.Collections.Specialized.OrderedDictionary]::new() - $atLimit.Add(('x' * 1024), 'value') + $atLimit.Add(('x' * 1022), 'value') $overLimit = [System.Collections.Specialized.OrderedDictionary]::new() - $overLimitKey = [char]::ConvertFromUtf32(0x1F600) * 1025 + $overLimitKey = [char]::ConvertFromUtf32(0x1F600) * 1023 $overLimit.Add($overLimitKey, [ordered]@{ nested = 'value' }) $atLimitYaml = ConvertTo-Yaml -InputObject $atLimit diff --git a/tests/Test-Yaml.Tests.ps1 b/tests/Test-Yaml.Tests.ps1 index 460d1f2..8d445da 100644 --- a/tests/Test-Yaml.Tests.ps1 +++ b/tests/Test-Yaml.Tests.ps1 @@ -43,16 +43,21 @@ Describe 'Test-Yaml' { ("['multi`n line': value]" | Test-Yaml) | Should -BeFalse ("[multi`n line: value]" | Test-Yaml) | Should -BeFalse - foreach ($quoted in @($false, $true)) { - foreach ($length in @(1024, 1025)) { - $text = 'k' * $length - $key = if ($quoted) { "'$text'" } else { $text } - foreach ($yaml in @( - "[$key`: value]", - "{$key`: value}" - )) { - ($yaml | Test-Yaml) | Should -Be ($length -eq 1024) - } + foreach ($case in @( + @{ Key = 'k' * 1024; Valid = $true } + @{ Key = 'k' * 1025; Valid = $false } + @{ Key = "'$('k' * 1022)'"; Valid = $true } + @{ Key = "'$('k' * 1023)'"; Valid = $false } + @{ Key = '"' + ('\u0061' * 170) + 'aa"'; Valid = $true } + @{ Key = '"' + ('\u0061' * 170) + 'aaa"'; Valid = $false } + @{ Key = '!local ' + ('k' * 1017); Valid = $true } + @{ Key = '!local ' + ('k' * 1018); Valid = $false } + )) { + foreach ($yaml in @( + "[$($case.Key)`: value]", + "{$($case.Key)`: value}" + )) { + ($yaml | Test-Yaml) | Should -Be $case.Valid } } @@ -61,6 +66,12 @@ Describe 'Test-Yaml' { ("{$key`: value}" | Test-Yaml) | Should -Be ($length -le 1024) } + + } + + It 'resolves non-specific tags before comparing representation keys' { + ("! x: one`n!!str x: two" | Test-Yaml) | Should -BeFalse + ("? ! [x]`n: one`n? !!seq [x]`n: two" | Test-Yaml) | Should -BeFalse } It 'accepts multiline keys in flow mappings' { From 587a4ae1dc82a54b26b8584fecf4259181436e73 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 24 Jul 2026 00:28:41 +0200 Subject: [PATCH 13/24] Share serialization node reservations Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/functions/private/ConvertTo-YamlNode.ps1 | 7 +++++ .../private/Get-YamlSerializationShape.ps1 | 14 +++++----- .../private/Read-YamlBoundedEnumerable.ps1 | 15 ++++++----- src/functions/public/ConvertTo-Yaml.ps1 | 1 + tests/ConvertTo-Yaml.Tests.ps1 | 26 ++++++++++++++++++- 5 files changed, 47 insertions(+), 16 deletions(-) diff --git a/src/functions/private/ConvertTo-YamlNode.ps1 b/src/functions/private/ConvertTo-YamlNode.ps1 index ba3f142..058bfeb 100644 --- a/src/functions/private/ConvertTo-YamlNode.ps1 +++ b/src/functions/private/ConvertTo-YamlNode.ps1 @@ -34,6 +34,7 @@ function ConvertTo-YamlNode { Child = $null KeyNode = $null Fingerprints = $null + Reserved = $false }) while ($stack.Count -gt 0) { @@ -44,6 +45,9 @@ function ConvertTo-YamlNode { "The object graph exceeds the configured depth limit of $($State.MaxDepth)." )) } + if ($frame.Reserved) { + $State.ReservedNodeCount-- + } $State.NodeCount++ if ($State.NodeCount -gt $State.MaxNodes) { throw (New-YamlSerializationException -ErrorId 'YamlNodeLimitExceeded' -Message ( @@ -122,6 +126,7 @@ function ConvertTo-YamlNode { Child = $null KeyNode = $null Fingerprints = $null + Reserved = $true }) continue } @@ -152,6 +157,7 @@ function ConvertTo-YamlNode { Child = $null KeyNode = $null Fingerprints = $null + Reserved = $true }) continue } @@ -180,6 +186,7 @@ function ConvertTo-YamlNode { Child = $null KeyNode = $null Fingerprints = $null + Reserved = $true }) continue } diff --git a/src/functions/private/Get-YamlSerializationShape.ps1 b/src/functions/private/Get-YamlSerializationShape.ps1 index e85529f..68221a2 100644 --- a/src/functions/private/Get-YamlSerializationShape.ps1 +++ b/src/functions/private/Get-YamlSerializationShape.ps1 @@ -198,11 +198,9 @@ function Get-YamlSerializationShape { } $entries = [System.Collections.Generic.List[object]]::new() if ($isDictionary) { - $remainingNodes = [System.Math]::Max(0, $State.MaxNodes - $State.NodeCount) - $maximumEntries = [int] [System.Math]::Floor($remainingNodes / 2.0) $rawEntries = Read-YamlBoundedEnumerable -Value $Value ` - -MaximumItems $maximumEntries -MaxNodes $State.MaxNodes -State $State ` - -DictionaryEntries -EnumsAsStrings:$EnumsAsStrings + -MaxNodes $State.MaxNodes -State $State -DictionaryEntries ` + -EnumsAsStrings:$EnumsAsStrings -NodesPerItem 2 foreach ($entry in $rawEntries) { $entries.Add([pscustomobject]@{ Key = [object] $entry.Key @@ -210,7 +208,8 @@ function Get-YamlSerializationShape { }) } } else { - if (($dataProperties.Count * 2) -gt ($State.MaxNodes - $State.NodeCount)) { + $availableNodes = $State.MaxNodes - $State.NodeCount - $State.ReservedNodeCount + if (($dataProperties.Count * 2) -gt $availableNodes) { throw (New-YamlSerializationException -ErrorId 'YamlNodeLimitExceeded' -Message ( "The object graph exceeds the configured limit of $($State.MaxNodes) nodes." )) @@ -220,6 +219,7 @@ function Get-YamlSerializationShape { -EnumsAsStrings:$EnumsAsStrings -InspectOnly $null = Get-YamlSerializationShape -Value $property.Value -State $State ` -EnumsAsStrings:$EnumsAsStrings -InspectOnly + $State.ReservedNodeCount += 2 $entries.Add([pscustomobject]@{ Key = [object] $property.Name Value = [object] $property.Value @@ -236,10 +236,8 @@ function Get-YamlSerializationShape { if ($InspectOnly) { return [pscustomobject]@{ Kind = 'Sequence'; Node = $null; Values = $null } } - $remainingNodes = [System.Math]::Max(0, $State.MaxNodes - $State.NodeCount) $items = Read-YamlBoundedEnumerable -Value $Value ` - -MaximumItems $remainingNodes -MaxNodes $State.MaxNodes -State $State ` - -EnumsAsStrings:$EnumsAsStrings + -MaxNodes $State.MaxNodes -State $State -EnumsAsStrings:$EnumsAsStrings return [pscustomobject]@{ Kind = 'Sequence' Node = $null diff --git a/src/functions/private/Read-YamlBoundedEnumerable.ps1 b/src/functions/private/Read-YamlBoundedEnumerable.ps1 index 37875b7..6bf9493 100644 --- a/src/functions/private/Read-YamlBoundedEnumerable.ps1 +++ b/src/functions/private/Read-YamlBoundedEnumerable.ps1 @@ -9,10 +9,6 @@ function Read-YamlBoundedEnumerable { [Parameter(Mandatory)] [System.Collections.IEnumerable] $Value, - [Parameter(Mandatory)] - [ValidateRange(0, 2147483647)] - [int] $MaximumItems, - [Parameter(Mandatory)] [int] $MaxNodes, @@ -21,11 +17,15 @@ function Read-YamlBoundedEnumerable { [switch] $DictionaryEntries, - [switch] $EnumsAsStrings + [switch] $EnumsAsStrings, + + [ValidateRange(1, 2)] + [int] $NodesPerItem = 1 ) + $availableNodes = $State.MaxNodes - $State.NodeCount - $State.ReservedNodeCount if ($Value -is [System.Collections.ICollection] -and - $Value.Count -gt $MaximumItems) { + ([long] $Value.Count * $NodesPerItem) -gt $availableNodes) { throw (New-YamlSerializationException -ErrorId 'YamlNodeLimitExceeded' -Message ( "The object graph exceeds the configured limit of $MaxNodes nodes." )) @@ -35,7 +35,7 @@ function Read-YamlBoundedEnumerable { $enumerator = $Value.GetEnumerator() try { while ($enumerator.MoveNext()) { - if ($items.Count -ge $MaximumItems) { + if (($State.NodeCount + $State.ReservedNodeCount + $NodesPerItem) -gt $MaxNodes) { throw (New-YamlSerializationException -ErrorId 'YamlNodeLimitExceeded' -Message ( "The object graph exceeds the configured limit of $MaxNodes nodes." )) @@ -50,6 +50,7 @@ function Read-YamlBoundedEnumerable { $null = Get-YamlSerializationShape -Value $item -State $State ` -EnumsAsStrings:$EnumsAsStrings -InspectOnly } + $State.ReservedNodeCount += $NodesPerItem $items.Add($item) } } finally { diff --git a/src/functions/public/ConvertTo-Yaml.ps1 b/src/functions/public/ConvertTo-Yaml.ps1 index f13e828..4f382cb 100644 --- a/src/functions/public/ConvertTo-Yaml.ps1 +++ b/src/functions/public/ConvertTo-Yaml.ps1 @@ -123,6 +123,7 @@ function ConvertTo-Yaml { FingerprintHasher = [System.Security.Cryptography.SHA256]::Create() Active = [System.Collections.Generic.HashSet[long]]::new() NodeCount = 0 + ReservedNodeCount = 0 MaxDepth = $Depth MaxNodes = $MaxNodes MaxScalarLength = $MaxScalarLength diff --git a/tests/ConvertTo-Yaml.Tests.ps1 b/tests/ConvertTo-Yaml.Tests.ps1 index 3f0de95..e0e70a9 100644 --- a/tests/ConvertTo-Yaml.Tests.ps1 +++ b/tests/ConvertTo-Yaml.Tests.ps1 @@ -58,6 +58,7 @@ namespace YamlTests { private readonly object[] values; public int GetEnumeratorCount { get; private set; } + public int MoveNextCount { get; private set; } public int DisposeCount { get; private set; } public OneShotEnumerable(object[] values) { this.values = values; } @@ -85,7 +86,12 @@ namespace YamlTests } public object Current { get { return values[index]; } } - public bool MoveNext() { index++; return index < values.Length; } + public bool MoveNext() + { + owner.MoveNextCount++; + index++; + return index < values.Length; + } public void Reset() { throw new NotSupportedException(); } public void Dispose() { owner.DisposeCount++; } } @@ -568,6 +574,24 @@ Describe 'ConvertTo-Yaml' { $source.DisposeCount | Should -Be 1 } + It 'shares the node budget while buffering nested enumerables' { + $children = [System.Collections.Generic.List[object]]::new() + $sources = [System.Collections.Generic.List[object]]::new() + foreach ($index in 1..9) { + $child = [YamlTests.OneShotEnumerable]::new([object[]] (1..9)) + $children.Add($child) + $sources.Add($child) + } + $root = [YamlTests.OneShotEnumerable]::new([object[]] $children.ToArray()) + $sources.Add($root) + + { ConvertTo-Yaml -InputObject $root -MaxNodes 20 } | + Should -Throw -ExpectedMessage '*configured limit of 20 nodes*' + ($sources | Measure-Object -Property MoveNextCount -Sum).Sum | + Should -BeLessOrEqual 22 + ($sources | Measure-Object -Property DisposeCount -Sum).Sum | Should -Be 3 + } + It 'stops and disposes enumerables at the first oversized scalar' { $source = [YamlTests.InfiniteEnumerable]::new('long') From 8108aa3cab606b5f432f5c8850f2eab0fe7ff401 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 24 Jul 2026 00:30:35 +0200 Subject: [PATCH 14/24] Require independent emit fixture oracles Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/Conformance.Tests.ps1 | 25 ++++++++++++ tests/tools/Invoke-YamlTestSuite.ps1 | 58 +++++++++++++++++++--------- 2 files changed, 65 insertions(+), 18 deletions(-) diff --git a/tests/Conformance.Tests.ps1 b/tests/Conformance.Tests.ps1 index 15a028b..8168ca3 100644 --- a/tests/Conformance.Tests.ps1 +++ b/tests/Conformance.Tests.ps1 @@ -270,6 +270,31 @@ ship-to: $mutatedResult.EmitYamlReason | Should -Be 'EmitYamlRepresentationMismatch' } + It 'uses JSON as the oracle for invalid-source emit fixtures' { + $mutatedSuitePath = Join-Path $TestDrive 'mutated-invalid-emit-fixtures' + $null = New-Item -Path $mutatedSuitePath -ItemType Directory -Force + Copy-Item -LiteralPath (Join-Path $suiteDataPath 'DK95') ` + -Destination $mutatedSuitePath -Recurse + foreach ($case in @('01', '06')) { + '--- altered' | Set-Content ` + -LiteralPath (Join-Path $mutatedSuitePath "DK95\$case\emit.yaml") ` + -Encoding utf8NoBOM + } + + $mutatedResults = @( + & $runnerPath -Path $mutatedSuitePath -CompareEmitYaml + ) + $invalidFixtureResults = @( + $mutatedResults | + Where-Object Case -In @('DK95/01', 'DK95/06') | + Sort-Object Case + ) + + @($invalidFixtureResults.EmitYamlResult) | Should -Be @('Fail', 'Fail') + @($invalidFixtureResults.EmitYamlReason | Select-Object -Unique) | + Should -Be @('EmitYamlRepresentationMismatch') + } + It 'accounts honestly for general module self-round-trips' { @($suiteResults | Where-Object SelfRoundTripResult -EQ 'Pass').Count | Should -Be 306 diff --git a/tests/tools/Invoke-YamlTestSuite.ps1 b/tests/tools/Invoke-YamlTestSuite.ps1 index f7294d9..e30fb45 100644 --- a/tests/tools/Invoke-YamlTestSuite.ps1 +++ b/tests/tools/Invoke-YamlTestSuite.ps1 @@ -812,6 +812,8 @@ foreach ($inputFile in $inputFiles) { $projectedJsonCanonical = $null $projectedReference = '' $projectionError = '' + $jsonOracleValues = $null + $jsonOracleCanonical = $null $eventExpected = $null $eventActual = $null $jsonExpected = $null @@ -879,6 +881,19 @@ foreach ($inputFile in $inputFiles) { } } + if ($hasJson -and ($CompareJson -or ($CompareEmitYaml -and $hasEmitYaml))) { + $expectedDocuments = Split-YamlSuiteJsonDocument -Text ( + [System.IO.File]::ReadAllText($jsonPath, [System.Text.UTF8Encoding]::new($false, $true)) + ) + $expectedValues = [System.Collections.Generic.List[object]]::new() + foreach ($document in $expectedDocuments) { + $expectedValues.Add((ConvertFrom-Json -InputObject $document -AsHashtable -NoEnumerate)) + } + $jsonOracleValues = [object[]] $expectedValues.ToArray() + $jsonOracleCanonical = ConvertTo-YamlSuiteCanonicalValue ` + -Value $jsonOracleValues -SortMappings + } + if ($CompareEvents -and $hasEvent) { if ($null -eq $representation -or $expectsError) { $eventResult = 'NotApplicable' @@ -906,22 +921,13 @@ foreach ($inputFile in $inputFiles) { if ($syntaxResult -eq 'PolicyDifference') { $jsonReason = $syntaxReason } if ($projectionError) { $jsonReason = $projectionError } } else { - $expectedDocuments = Split-YamlSuiteJsonDocument -Text ( - [System.IO.File]::ReadAllText($jsonPath, [System.Text.UTF8Encoding]::new($false, $true)) - ) - $expectedValues = [System.Collections.Generic.List[object]]::new() - foreach ($document in $expectedDocuments) { - $expectedValues.Add((ConvertFrom-Json -InputObject $document -AsHashtable -NoEnumerate)) - } - $expectedCanonical = ConvertTo-YamlSuiteCanonicalValue ` - -Value ([object[]] $expectedValues.ToArray()) -SortMappings - $jsonExpected = $expectedCanonical + $jsonExpected = $jsonOracleCanonical $jsonActual = $projectedJsonCanonical - if ($projectedJsonCanonical -ceq $expectedCanonical) { + if ($projectedJsonCanonical -ceq $jsonOracleCanonical) { $jsonResult = 'Pass' } else { $reason = Get-YamlSuiteJsonPolicyReason ` - -ExpectedValues ([object[]] $expectedValues.ToArray()) ` + -ExpectedValues $jsonOracleValues ` -ActualValues ([object[]] $projectedValues) if ($reason) { $jsonResult = 'PolicyDifference' @@ -990,14 +996,30 @@ foreach ($inputFile in $inputFiles) { $fixtureReference = ConvertTo-YamlSuiteReferenceSignature ` -Value ([object[]] $fixtureValues) - if ($null -ne $projectedCanonical -and - ($fixtureCanonical -cne $projectedCanonical -or - $fixtureReference -cne $projectedReference)) { + $fixtureOracleCanonical = $null + $fixtureOracleActual = $fixtureCanonical + $fixtureReferenceMismatch = $false + if ($null -ne $projectedCanonical) { + $fixtureOracleCanonical = $projectedCanonical + $fixtureReferenceMismatch = $fixtureReference -cne $projectedReference + } elseif ($null -ne $jsonOracleCanonical) { + $fixtureOracleCanonical = $jsonOracleCanonical + $fixtureOracleActual = ConvertTo-YamlSuiteCanonicalValue ` + -Value ([object[]] $fixtureValues) -SortMappings + } + + if ($null -eq $fixtureOracleCanonical) { + $emitYamlResult = 'Fail' + $emitYamlReason = 'EmitYamlOracleUnavailable' + } elseif ($fixtureOracleActual -cne $fixtureOracleCanonical -or + $fixtureReferenceMismatch) { $emitYamlResult = 'Fail' $emitYamlReason = 'EmitYamlRepresentationMismatch' - $emitYamlExpected = $projectedCanonical - $emitYamlActual = $fixtureCanonical - $emitYamlExpectedReference = $projectedReference + $emitYamlExpected = $fixtureOracleCanonical + $emitYamlActual = $fixtureOracleActual + if ($null -ne $projectedCanonical) { + $emitYamlExpectedReference = $projectedReference + } $emitYamlActualReference = $fixtureReference } else { $fixtureEmittedDocuments = [System.Collections.Generic.List[string]]::new() From dc8bda3ebb2889e3d4828aba3ed979850b817e1b Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 24 Jul 2026 00:32:10 +0200 Subject: [PATCH 15/24] Stabilize validation under coverage Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/functions/private/ConvertTo-YamlFlowText.ps1 | 4 +--- src/functions/private/Get-YamlEmissionImplicitKeyLength.ps1 | 3 --- src/functions/private/Read-YamlFlowNode.ps1 | 4 ++-- src/functions/private/Write-YamlNodeText.ps1 | 3 +-- tests/ConvertFrom-Yaml.Tests.ps1 | 2 +- 5 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/functions/private/ConvertTo-YamlFlowText.ps1 b/src/functions/private/ConvertTo-YamlFlowText.ps1 index 47060dd..2754cb2 100644 --- a/src/functions/private/ConvertTo-YamlFlowText.ps1 +++ b/src/functions/private/ConvertTo-YamlFlowText.ps1 @@ -132,10 +132,8 @@ function ConvertTo-YamlFlowText { continue } if ($frame.State -eq 'MappingValue') { - $keyNode = $frame.Node.Entries[$frame.Index].Key $explicitKey = ( - Get-YamlEmissionImplicitKeyLength -Node $keyNode ` - -RenderedText $frame.KeyText + Get-YamlEmissionImplicitKeyLength -RenderedText $frame.KeyText ) -gt 1024 $keyPrefix = if ($explicitKey) { '? ' } else { '' } $frame.Parts.Add("$keyPrefix$($frame.KeyText)`: $($frame.Child.Value)") diff --git a/src/functions/private/Get-YamlEmissionImplicitKeyLength.ps1 b/src/functions/private/Get-YamlEmissionImplicitKeyLength.ps1 index 4962d36..af52971 100644 --- a/src/functions/private/Get-YamlEmissionImplicitKeyLength.ps1 +++ b/src/functions/private/Get-YamlEmissionImplicitKeyLength.ps1 @@ -6,9 +6,6 @@ function Get-YamlEmissionImplicitKeyLength { [CmdletBinding()] [OutputType([int])] param ( - [Parameter(Mandatory)] - [pscustomobject] $Node, - [Parameter(Mandatory)] [string] $RenderedText ) diff --git a/src/functions/private/Read-YamlFlowNode.ps1 b/src/functions/private/Read-YamlFlowNode.ps1 index 021328e..b567ec9 100644 --- a/src/functions/private/Read-YamlFlowNode.ps1 +++ b/src/functions/private/Read-YamlFlowNode.ps1 @@ -563,7 +563,7 @@ function Read-YamlFlowNode { continue } if ($character -eq "`n") { - $pendingWhiteSpace.Clear() | Out-Null + $null = $pendingWhiteSpace.Clear() $pendingBreaks = 0 while ($Cursor.Index -lt $Context.Text.Length -and $Context.Text[$Cursor.Index] -ceq "`n") { @@ -598,7 +598,7 @@ function Read-YamlFlowNode { [void] $builder.Append($pendingWhiteSpace) } $pendingBreaks = 0 - $pendingWhiteSpace.Clear() | Out-Null + $null = $pendingWhiteSpace.Clear() [void] $builder.Append($character) if ($builder.Length -gt $Context.MaxScalarLength) { throw (New-YamlException -Start $start -End $start -ErrorId 'YamlScalarLimitExceeded' -Message ( diff --git a/src/functions/private/Write-YamlNodeText.ps1 b/src/functions/private/Write-YamlNodeText.ps1 index cc67624..4d6fe37 100644 --- a/src/functions/private/Write-YamlNodeText.ps1 +++ b/src/functions/private/Write-YamlNodeText.ps1 @@ -46,8 +46,7 @@ function Write-YamlNodeText { $keyText = ConvertTo-YamlFlowText -Node $task.Entry.Key ` -EmittedReferences $EmittedReferences $explicitKey = ( - Get-YamlEmissionImplicitKeyLength -Node $task.Entry.Key ` - -RenderedText $keyText + Get-YamlEmissionImplicitKeyLength -RenderedText $keyText ) -gt 1024 if ($explicitKey) { $spaces = ' ' * ($task.Level * $Indent) diff --git a/tests/ConvertFrom-Yaml.Tests.ps1 b/tests/ConvertFrom-Yaml.Tests.ps1 index 7a69e3d..463dd6d 100644 --- a/tests/ConvertFrom-Yaml.Tests.ps1 +++ b/tests/ConvertFrom-Yaml.Tests.ps1 @@ -523,7 +523,7 @@ date: !!timestamp 2001-12-14 $tagAmplification = ( @("%TAG !e! tag:example.test,$prefix", '---') + $taggedItems ) -join "`n" - $hugeInteger = '9' * 320000 + $hugeInteger = '9' * 32000 ($tagAmplification | Test-Yaml -MaxTagLength 25000) | Should -BeFalse From a24c58ad84c9066d9b3a47dd74a76684eb3b22e6 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 24 Jul 2026 01:04:11 +0200 Subject: [PATCH 16/24] Close final representation edge cases Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../private/Assert-YamlNoByteOrderMark.ps1 | 21 +++++++++++++++++++ .../private/ConvertFrom-YamlByteOrderMark.ps1 | 9 +------- .../private/ConvertTo-YamlQuotedText.ps1 | 2 +- .../private/Get-YamlEffectiveTag.ps1 | 5 ++--- src/functions/private/Read-YamlBlockKey.ps1 | 1 + .../private/Read-YamlBlockMapping.ps1 | 3 ++- .../private/Read-YamlBlockScalar.ps1 | 1 + src/functions/private/Read-YamlFlowNode.ps1 | 8 +++++++ .../private/Read-YamlNodeProperty.ps1 | 4 ++++ .../private/Read-YamlPlainScalar.ps1 | 2 ++ src/functions/private/Resolve-YamlTag.ps1 | 9 ++++++-- tests/ConvertFrom-Yaml.Tests.ps1 | 2 ++ tests/ConvertTo-Yaml.Tests.ps1 | 9 ++++++++ tests/Test-Yaml.Tests.ps1 | 5 +++++ 14 files changed, 66 insertions(+), 15 deletions(-) create mode 100644 src/functions/private/Assert-YamlNoByteOrderMark.ps1 diff --git a/src/functions/private/Assert-YamlNoByteOrderMark.ps1 b/src/functions/private/Assert-YamlNoByteOrderMark.ps1 new file mode 100644 index 0000000..1171d22 --- /dev/null +++ b/src/functions/private/Assert-YamlNoByteOrderMark.ps1 @@ -0,0 +1,21 @@ +function Assert-YamlNoByteOrderMark { + <# + .SYNOPSIS + Rejects a raw byte order mark outside a quoted scalar or document prefix. + #> + [CmdletBinding()] + param ( + [Parameter(Mandatory)] + [AllowEmptyString()] + [string] $Text, + + [Parameter(Mandatory)] + [pscustomobject] $Mark + ) + + if ($Text.IndexOf([char] 0xFEFF) -ge 0) { + throw (New-YamlException -Start $Mark -End $Mark -ErrorId 'YamlInvalidByteOrderMark' -Message ( + 'A raw YAML byte order mark is only allowed in a quoted scalar or document prefix.' + )) + } +} diff --git a/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 b/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 index 8ede89b..52f2082 100644 --- a/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 +++ b/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 @@ -32,14 +32,7 @@ function ConvertFrom-YamlByteOrderMark { continue } - $before = $Text.Substring(0, $index) - $line = ([regex]::Matches($before, "`n")).Count - $lastBreak = $before.LastIndexOf("`n", [System.StringComparison]::Ordinal) - $column = if ($lastBreak -lt 0) { $before.Length } else { $before.Length - $lastBreak - 1 } - $mark = New-YamlMark -Index $index -Line $line -Column $column - throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidByteOrderMark' -Message ( - 'A YAML byte order mark is only allowed at the start of the stream or a document prefix.' - )) + [void] $builder.Append($Text[$index]) } return $builder.ToString() diff --git a/src/functions/private/ConvertTo-YamlQuotedText.ps1 b/src/functions/private/ConvertTo-YamlQuotedText.ps1 index e6deb67..4090cfb 100644 --- a/src/functions/private/ConvertTo-YamlQuotedText.ps1 +++ b/src/functions/private/ConvertTo-YamlQuotedText.ps1 @@ -50,7 +50,7 @@ function ConvertTo-YamlQuotedText { [void] $builder.Append('\"') } elseif ($code -eq 92) { [void] $builder.Append('\\') - } elseif ($code -lt 0x20 -or $code -eq 0x7F -or + } elseif ($code -lt 0x20 -or $code -eq 0x7F -or $code -eq 0xFEFF -or $code -eq 0xFFFE -or $code -eq 0xFFFF -or ($code -ge 0x80 -and $code -le 0x9F)) { [void] $builder.Append(('\u{0:X4}' -f $code)) diff --git a/src/functions/private/Get-YamlEffectiveTag.ps1 b/src/functions/private/Get-YamlEffectiveTag.ps1 index 5619941..4bb7a65 100644 --- a/src/functions/private/Get-YamlEffectiveTag.ps1 +++ b/src/functions/private/Get-YamlEffectiveTag.ps1 @@ -14,8 +14,7 @@ function Get-YamlEffectiveTag { [object] $Value ) - if (-not [string]::IsNullOrEmpty($Node.Tag) -and - -not $Node.Tag.Equals('!', [System.StringComparison]::Ordinal)) { + if (-not [string]::IsNullOrEmpty($Node.Tag)) { return $Node.Tag } if ($Node.Kind.Equals('Sequence', [System.StringComparison]::Ordinal)) { @@ -24,7 +23,7 @@ function Get-YamlEffectiveTag { if ($Node.Kind.Equals('Mapping', [System.StringComparison]::Ordinal)) { return 'tag:yaml.org,2002:map' } - if ($Node.Tag.Equals('!', [System.StringComparison]::Ordinal)) { + if ($Node.HasUnknownTag) { return 'tag:yaml.org,2002:str' } diff --git a/src/functions/private/Read-YamlBlockKey.ps1 b/src/functions/private/Read-YamlBlockKey.ps1 index de8e861..9b53d7d 100644 --- a/src/functions/private/Read-YamlBlockKey.ps1 +++ b/src/functions/private/Read-YamlBlockKey.ps1 @@ -62,6 +62,7 @@ function Read-YamlBlockKey { return $node } + Assert-YamlNoByteOrderMark -Text $rest -Mark $start $first = $rest[0] if ($first -in @(',', '[', ']', '{', '}', '#', '&', '*', '!', '|', '>', "'", '"', '%', '@', '`') -or ($first -in @('-', '?', ':') -and ( diff --git a/src/functions/private/Read-YamlBlockMapping.ps1 b/src/functions/private/Read-YamlBlockMapping.ps1 index ab8c0d1..c9d3aac 100644 --- a/src/functions/private/Read-YamlBlockMapping.ps1 +++ b/src/functions/private/Read-YamlBlockMapping.ps1 @@ -191,7 +191,8 @@ function Read-YamlBlockMapping { $key = Read-YamlBlockKey -Context $Context -Text $keyText -Line $lineNumber ` -Column $contentColumn -Depth ($Depth + 1) } - if ($colon -gt 0 -and (Get-YamlRuneCount -Text $keyText) -gt 1024) { + if ($colon -gt 0 -and + (Get-YamlRuneCount -Text $content.Substring(0, $colon)) -gt 1024) { $mark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $contentColumn + $colon) ` -Line $lineNumber -Column ($contentColumn + $colon) throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidImplicitKey' -Message ( diff --git a/src/functions/private/Read-YamlBlockScalar.ps1 b/src/functions/private/Read-YamlBlockScalar.ps1 index 1c9320f..0824982 100644 --- a/src/functions/private/Read-YamlBlockScalar.ps1 +++ b/src/functions/private/Read-YamlBlockScalar.ps1 @@ -97,6 +97,7 @@ function Read-YamlBlockScalar { $decodedLength = 0 while ($Context.LineIndex -lt $Context.Lines.Count) { $line = $Context.Lines[$Context.LineIndex] + Assert-YamlNoByteOrderMark -Text $line -Mark $start if ($line -match '^(?:---|\.\.\.)(?:[ \t]|$)') { break } diff --git a/src/functions/private/Read-YamlFlowNode.ps1 b/src/functions/private/Read-YamlFlowNode.ps1 index b567ec9..f1de657 100644 --- a/src/functions/private/Read-YamlFlowNode.ps1 +++ b/src/functions/private/Read-YamlFlowNode.ps1 @@ -90,6 +90,7 @@ function Read-YamlFlowNode { Move-YamlCursor -Cursor $Cursor -Context $Context } $anchor = $Context.Text.Substring($anchorStart, $Cursor.Index - $anchorStart) + Assert-YamlNoByteOrderMark -Text $anchor -Mark $start if ([string]::IsNullOrEmpty($anchor)) { throw (New-YamlException -Start $start -End $start -ErrorId 'YamlInvalidAnchor' -Message ( 'A YAML anchor name is missing.' @@ -137,6 +138,7 @@ function Read-YamlFlowNode { Move-YamlCursor -Cursor $Cursor -Context $Context } $alias = $Context.Text.Substring($aliasStart, $Cursor.Index - $aliasStart) + Assert-YamlNoByteOrderMark -Text $alias -Mark $start $Context.AliasCount++ if ($Context.AliasCount -gt $Context.MaxAliases) { throw (New-YamlException -Start $start -End $start -ErrorId 'YamlAliasLimitExceeded' -Message ( @@ -544,6 +546,12 @@ function Read-YamlFlowNode { } while ($Cursor.Index -lt $Context.Text.Length) { $character = $Context.Text[$Cursor.Index] + if ($character -ceq [char] 0xFEFF) { + $mark = New-YamlMark -Index $Cursor.Index -Line $Cursor.Line -Column $Cursor.Column + throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidByteOrderMark' -Message ( + 'A raw YAML byte order mark is only allowed in a quoted scalar or document prefix.' + )) + } if ($character -in @(',', '[', ']', '{', '}')) { break } diff --git a/src/functions/private/Read-YamlNodeProperty.ps1 b/src/functions/private/Read-YamlNodeProperty.ps1 index fcbd5b5..e1b46d4 100644 --- a/src/functions/private/Read-YamlNodeProperty.ps1 +++ b/src/functions/private/Read-YamlNodeProperty.ps1 @@ -84,6 +84,10 @@ function Read-YamlNodeProperty { $position++ } $anchor = $Text.Substring($start, $position - $start) + Assert-YamlNoByteOrderMark -Text $anchor -Mark ( + New-YamlMark -Index ($Context.LineStarts[$Line] + $Column + $start) -Line $Line ` + -Column ($Column + $start) + ) if ([string]::IsNullOrEmpty($anchor) -or $anchor.IndexOfAny(@('[', ']', '{', '}', ',')) -ge 0) { $mark = New-YamlMark -Index ($Context.LineStarts[$Line] + $Column + $start - 1) -Line $Line ` -Column ($Column + $start - 1) diff --git a/src/functions/private/Read-YamlPlainScalar.ps1 b/src/functions/private/Read-YamlPlainScalar.ps1 index cf22818..edc9769 100644 --- a/src/functions/private/Read-YamlPlainScalar.ps1 +++ b/src/functions/private/Read-YamlPlainScalar.ps1 @@ -37,6 +37,7 @@ function Read-YamlPlainScalar { $parts = [System.Collections.Generic.List[object]]::new() $firstComment = Find-YamlCommentStart -Text $FirstText $firstValue = (Get-YamlContentWithoutComment -Text $FirstText).Trim(' ', "`t") + Assert-YamlNoByteOrderMark -Text $firstValue -Mark $start $firstCharacter = if ($firstValue.Length -gt 0) { $firstValue[0] } else { [char] 0 } $forbiddenFirst = $firstCharacter -in @( ',', '[', ']', '{', '}', '#', '&', '*', '!', '|', '>', "'", '"', '%', '@', '`' @@ -91,6 +92,7 @@ function Read-YamlPlainScalar { break } $trimmedContent = $content.Trim(' ', "`t") + Assert-YamlNoByteOrderMark -Text $trimmedContent -Mark $start $separatorLength = if ($pendingBreaks -gt 0) { $pendingBreaks } else { 1 } if ($decodedLength + $separatorLength + $trimmedContent.Length -gt $Context.MaxScalarLength) { diff --git a/src/functions/private/Resolve-YamlTag.ps1 b/src/functions/private/Resolve-YamlTag.ps1 index 7483f69..b98ba5b 100644 --- a/src/functions/private/Resolve-YamlTag.ps1 +++ b/src/functions/private/Resolve-YamlTag.ps1 @@ -22,6 +22,7 @@ function Resolve-YamlTag { )) } + $isNonSpecific = $Token.Equals('!', [System.StringComparison]::Ordinal) if ($Token.StartsWith('!<', [System.StringComparison]::Ordinal)) { if (-not $Token.EndsWith('>', [System.StringComparison]::Ordinal) -or $Token.Length -lt 4) { throw (New-YamlException -Start $Mark -End $Mark -ErrorId 'YamlInvalidTag' -Message ( @@ -67,7 +68,11 @@ function Resolve-YamlTag { } } - $expanded = ConvertFrom-YamlTagUriEscape -Text ($prefix + $suffix) -Mark $Mark -Token $Token + $expanded = if ($isNonSpecific) { + '' + } else { + ConvertFrom-YamlTagUriEscape -Text ($prefix + $suffix) -Mark $Mark -Token $Token + } $expandedLength = $expanded.Length if ($expandedLength -gt $Context.MaxTagLength) { throw (New-YamlException -Start $Mark -End $Mark -ErrorId 'YamlTagLimitExceeded' -Message ( @@ -98,6 +103,6 @@ function Resolve-YamlTag { [pscustomobject]@{ Tag = $expanded - IsUnknown = -not $known + IsUnknown = $isNonSpecific -or -not $known } } diff --git a/tests/ConvertFrom-Yaml.Tests.ps1 b/tests/ConvertFrom-Yaml.Tests.ps1 index 463dd6d..6b9bb49 100644 --- a/tests/ConvertFrom-Yaml.Tests.ps1 +++ b/tests/ConvertFrom-Yaml.Tests.ps1 @@ -213,6 +213,8 @@ folded: > ("foo${bom}bar" | Test-Yaml) | Should -BeFalse ("---`n${bom}value" | Test-Yaml) | Should -BeFalse ("---`none`n...`n${bom}# comment`ntwo" | Test-Yaml) | Should -BeFalse + ('"foo' + $bom + 'bar"' | ConvertFrom-Yaml) | Should -Be "foo${bom}bar" + ("'foo${bom}bar'" | ConvertFrom-Yaml) | Should -Be "foo${bom}bar" } } diff --git a/tests/ConvertTo-Yaml.Tests.ps1 b/tests/ConvertTo-Yaml.Tests.ps1 index e0e70a9..2796fca 100644 --- a/tests/ConvertTo-Yaml.Tests.ps1 +++ b/tests/ConvertTo-Yaml.Tests.ps1 @@ -171,6 +171,15 @@ Describe 'ConvertTo-Yaml' { $result['multiline'] | Should -Be $inputObject.multiline } + It 'escapes byte order marks as quoted scalar content' { + $value = "foo$([char] 0xFEFF)bar" + + $yaml = ConvertTo-Yaml -InputObject $value + + $yaml.TrimEnd("`n") | Should -Be '"foo\uFEFFbar"' + ($yaml | ConvertFrom-Yaml) | Should -Be $value + } + It 'emits only valid YAML characters and rejects malformed UTF-16 input' { $noncharacters = ([string] [char] 0xFFFE) + [char] 0xFFFF $yaml = ConvertTo-Yaml -InputObject $noncharacters diff --git a/tests/Test-Yaml.Tests.ps1 b/tests/Test-Yaml.Tests.ps1 index 8d445da..7a34d35 100644 --- a/tests/Test-Yaml.Tests.ps1 +++ b/tests/Test-Yaml.Tests.ps1 @@ -67,11 +67,16 @@ Describe 'Test-Yaml' { Should -Be ($length -le 1024) } + ((('k' * 1023) + ' : value') | Test-Yaml) | Should -BeTrue + ((('k' * 1024) + ' : value') | Test-Yaml) | Should -BeFalse + } It 'resolves non-specific tags before comparing representation keys' { ("! x: one`n!!str x: two" | Test-Yaml) | Should -BeFalse ("? ! [x]`n: one`n? !!seq [x]`n: two" | Test-Yaml) | Should -BeFalse + ("! x: one`n! x: two" | Test-Yaml) | Should -BeTrue + ("! x: one`n! x: two" | Test-Yaml) | Should -BeFalse } It 'accepts multiline keys in flow mappings' { From 800e0ed06575df8cd8aa89b56bf744bfac52ad55 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 24 Jul 2026 01:27:27 +0200 Subject: [PATCH 17/24] Consume BOMs in parser document state Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../private/ConvertFrom-YamlByteOrderMark.ps1 | 39 ------------------ .../private/New-YamlReaderContext.ps1 | 1 - .../private/Read-YamlBlockMapping.ps1 | 3 +- src/functions/private/Read-YamlBlockNode.ps1 | 6 ++- .../private/Read-YamlBlockScalar.ps1 | 5 ++- .../private/Read-YamlBlockSequence.ps1 | 7 +++- .../private/Read-YamlDirectiveBlock.ps1 | 1 + .../Read-YamlDocumentByteOrderMark.ps1 | 40 +++++++++++++++++++ .../private/Read-YamlPlainScalar.ps1 | 3 +- src/functions/private/Read-YamlStreamCore.ps1 | 10 +++-- .../Test-YamlDocumentByteOrderMark.ps1 | 25 ++++++++++++ .../private/Test-YamlDocumentPrefix.ps1 | 13 ++++-- tests/ConvertFrom-Yaml.Tests.ps1 | 8 ++++ 13 files changed, 107 insertions(+), 54 deletions(-) delete mode 100644 src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 create mode 100644 src/functions/private/Read-YamlDocumentByteOrderMark.ps1 create mode 100644 src/functions/private/Test-YamlDocumentByteOrderMark.ps1 diff --git a/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 b/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 deleted file mode 100644 index 52f2082..0000000 --- a/src/functions/private/ConvertFrom-YamlByteOrderMark.ps1 +++ /dev/null @@ -1,39 +0,0 @@ -function ConvertFrom-YamlByteOrderMark { - <# - .SYNOPSIS - Consumes byte order marks at YAML stream and explicit document boundaries. - #> - [CmdletBinding()] - [OutputType([string])] - param ( - [Parameter(Mandatory)] - [AllowEmptyString()] - [string] $Text - ) - - if ($Text.IndexOf([char] 0xFEFF) -lt 0) { - return $Text - } - - $builder = [System.Text.StringBuilder]::new($Text.Length) - for ($index = 0; $index -lt $Text.Length; $index++) { - if ($Text[$index] -cne [char] 0xFEFF) { - [void] $builder.Append($Text[$index]) - continue - } - - $atStreamStart = $index -eq 0 - $atLineStart = $index -gt 0 -and $Text[$index - 1] -ceq "`n" - $beforeDocumentPrefix = $atLineStart -and ( - Test-YamlDocumentPrefix -Text $Text -Index ($index + 1) - ) - - if ($atStreamStart -or $beforeDocumentPrefix) { - continue - } - - [void] $builder.Append($Text[$index]) - } - - return $builder.ToString() -} diff --git a/src/functions/private/New-YamlReaderContext.ps1 b/src/functions/private/New-YamlReaderContext.ps1 index 51808be..de48ca6 100644 --- a/src/functions/private/New-YamlReaderContext.ps1 +++ b/src/functions/private/New-YamlReaderContext.ps1 @@ -37,7 +37,6 @@ function New-YamlReaderContext { ) $text = $Yaml.Replace("`r`n", "`n").Replace("`r", "`n") - $text = ConvertFrom-YamlByteOrderMark -Text $text Assert-YamlText -Yaml $text $lines = [System.Text.RegularExpressions.Regex]::Split($text, "`n") $lineStarts = [int[]]::new($lines.Count) diff --git a/src/functions/private/Read-YamlBlockMapping.ps1 b/src/functions/private/Read-YamlBlockMapping.ps1 index c9d3aac..562ee11 100644 --- a/src/functions/private/Read-YamlBlockMapping.ps1 +++ b/src/functions/private/Read-YamlBlockMapping.ps1 @@ -49,7 +49,8 @@ function Read-YamlBlockMapping { $lineNumber = $Context.LineIndex $line = $Context.Lines[$lineNumber] $trimmed = $line.TrimStart(' ', "`t") - if ($line -match '^(?:---|\.\.\.)(?:[ \t]|$)') { + if ($line -match '^(?:---|\.\.\.)(?:[ \t]|$)' -or + (Test-YamlDocumentByteOrderMark -Context $Context -RequireDocumentStart)) { break } if ($trimmed.Length -eq 0 -or $trimmed.StartsWith('#', [System.StringComparison]::Ordinal)) { diff --git a/src/functions/private/Read-YamlBlockNode.ps1 b/src/functions/private/Read-YamlBlockNode.ps1 index cdc39a4..38a24a5 100644 --- a/src/functions/private/Read-YamlBlockNode.ps1 +++ b/src/functions/private/Read-YamlBlockNode.ps1 @@ -44,7 +44,8 @@ function Read-YamlBlockNode { -HasUnknownTag $PendingUnknownTag -Anchor $PendingAnchor } $line = $Context.Lines[$Context.LineIndex] - if ($line -match '^(?:---|\.\.\.)(?:[ \t]|$)') { + if ($line -match '^(?:---|\.\.\.)(?:[ \t]|$)' -or + (Test-YamlDocumentByteOrderMark -Context $Context -RequireDocumentStart)) { $mark = New-YamlMark -Index $Context.LineStarts[$Context.LineIndex] -Line $Context.LineIndex -Column 0 return New-YamlEmptyScalar -Context $Context -Depth $Depth -Mark $mark -Tag $PendingTag ` -HasUnknownTag $PendingUnknownTag -Anchor $PendingAnchor @@ -119,7 +120,8 @@ function Read-YamlBlockNode { $Context.LineIndex++ Skip-YamlBlockTrivia -Context $Context if ($Context.LineIndex -ge $Context.Lines.Count -or - $Context.Lines[$Context.LineIndex] -match '^(?:---|\.\.\.)(?:[ \t]|$)') { + $Context.Lines[$Context.LineIndex] -match '^(?:---|\.\.\.)(?:[ \t]|$)' -or + (Test-YamlDocumentByteOrderMark -Context $Context -RequireDocumentStart)) { $mark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $contentColumn) ` -Line $lineNumber -Column $contentColumn return New-YamlEmptyScalar -Context $Context -Depth $Depth -Mark $mark -Tag $tag ` diff --git a/src/functions/private/Read-YamlBlockScalar.ps1 b/src/functions/private/Read-YamlBlockScalar.ps1 index 0824982..b3d984d 100644 --- a/src/functions/private/Read-YamlBlockScalar.ps1 +++ b/src/functions/private/Read-YamlBlockScalar.ps1 @@ -97,10 +97,11 @@ function Read-YamlBlockScalar { $decodedLength = 0 while ($Context.LineIndex -lt $Context.Lines.Count) { $line = $Context.Lines[$Context.LineIndex] - Assert-YamlNoByteOrderMark -Text $line -Mark $start - if ($line -match '^(?:---|\.\.\.)(?:[ \t]|$)') { + if ($line -match '^(?:---|\.\.\.)(?:[ \t]|$)' -or + (Test-YamlDocumentByteOrderMark -Context $Context -RequireDocumentStart)) { break } + Assert-YamlNoByteOrderMark -Text $line -Mark $start if ($Context.LineIndex -eq $Context.Lines.Count - 1 -and $line.Length -eq 0 -and $Context.Text.EndsWith("`n", [System.StringComparison]::Ordinal)) { diff --git a/src/functions/private/Read-YamlBlockSequence.ps1 b/src/functions/private/Read-YamlBlockSequence.ps1 index 3c272b4..cb8aba7 100644 --- a/src/functions/private/Read-YamlBlockSequence.ps1 +++ b/src/functions/private/Read-YamlBlockSequence.ps1 @@ -47,7 +47,8 @@ function Read-YamlBlockSequence { } else { $line = $Context.Lines[$Context.LineIndex] $trimmed = $line.TrimStart(' ', "`t") - if ($line -match '^(?:---|\.\.\.)(?:[ \t]|$)') { + if ($line -match '^(?:---|\.\.\.)(?:[ \t]|$)' -or + (Test-YamlDocumentByteOrderMark -Context $Context -RequireDocumentStart)) { break } if ($trimmed.Length -eq 0 -or $trimmed.StartsWith('#', [System.StringComparison]::Ordinal)) { @@ -80,7 +81,9 @@ function Read-YamlBlockSequence { } else { $nextLine = $Context.Lines[$Context.LineIndex] $nextIndent = Get-YamlIndent -Line $nextLine -LineNumber $Context.LineIndex -Context $Context - if ($nextIndent -le $Indent -or $nextLine -match '^(?:---|\.\.\.)(?:[ \t]|$)') { + if ($nextIndent -le $Indent -or + $nextLine -match '^(?:---|\.\.\.)(?:[ \t]|$)' -or + (Test-YamlDocumentByteOrderMark -Context $Context -RequireDocumentStart)) { $mark = New-YamlMark -Index ($Context.LineStarts[$line] + $itemColumn) -Line $line ` -Column $itemColumn $item = New-YamlEmptyScalar -Context $Context -Depth ($Depth + 1) -Mark $mark diff --git a/src/functions/private/Read-YamlDirectiveBlock.ps1 b/src/functions/private/Read-YamlDirectiveBlock.ps1 index 627be01..d616eea 100644 --- a/src/functions/private/Read-YamlDirectiveBlock.ps1 +++ b/src/functions/private/Read-YamlDirectiveBlock.ps1 @@ -30,6 +30,7 @@ function Read-YamlDirectiveBlock { $directive = $Context.Lines[$Context.LineIndex] $mark = New-YamlMark -Index $Context.LineStarts[$Context.LineIndex] ` -Line $Context.LineIndex -Column 0 + Assert-YamlNoByteOrderMark -Text $directive -Mark $mark if ($directive -cmatch '^%YAML(?:[ \t]|$)') { if ($yamlDirectiveSeen -or $directive -cnotmatch ( '^%YAML[ \t]+([0-9]+)\.([0-9]+)(?:[ \t]+#.*)?$' diff --git a/src/functions/private/Read-YamlDocumentByteOrderMark.ps1 b/src/functions/private/Read-YamlDocumentByteOrderMark.ps1 new file mode 100644 index 0000000..64334d7 --- /dev/null +++ b/src/functions/private/Read-YamlDocumentByteOrderMark.ps1 @@ -0,0 +1,40 @@ +function Read-YamlDocumentByteOrderMark { + <# + .SYNOPSIS + Consumes a byte order mark while scanning an actual document prefix. + #> + [Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSUseShouldProcessForStateChangingFunctions', '', + Justification = 'Advances an in-memory parser context.' + )] + [CmdletBinding()] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Context, + + [switch] $RequireDocumentStart + ) + + if ($Context.LineIndex -ge $Context.Lines.Count) { + return + } + $index = $Context.LineStarts[$Context.LineIndex] + $atStreamStart = $index -eq 0 + if (-not $atStreamStart -and + -not (Test-YamlDocumentByteOrderMark -Context $Context ` + -RequireDocumentStart:$RequireDocumentStart)) { + return + } + if (-not $Context.Lines[$Context.LineIndex].StartsWith( + [string] [char] 0xFEFF, + [System.StringComparison]::Ordinal + )) { + return + } + + $Context.Text = $Context.Text.Remove($index, 1) + $Context.Lines[$Context.LineIndex] = $Context.Lines[$Context.LineIndex].Substring(1) + for ($line = $Context.LineIndex + 1; $line -lt $Context.LineStarts.Count; $line++) { + $Context.LineStarts[$line]-- + } +} diff --git a/src/functions/private/Read-YamlPlainScalar.ps1 b/src/functions/private/Read-YamlPlainScalar.ps1 index edc9769..d804442 100644 --- a/src/functions/private/Read-YamlPlainScalar.ps1 +++ b/src/functions/private/Read-YamlPlainScalar.ps1 @@ -70,7 +70,8 @@ function Read-YamlPlainScalar { while ($firstComment -lt 0 -and $Context.LineIndex -lt $Context.Lines.Count) { $line = $Context.Lines[$Context.LineIndex] $trimmed = $line.TrimStart(' ', "`t") - if ($line -match '^(?:---|\.\.\.)(?:[ \t]|$)') { + if ($line -match '^(?:---|\.\.\.)(?:[ \t]|$)' -or + (Test-YamlDocumentByteOrderMark -Context $Context -RequireDocumentStart)) { break } if ($trimmed.Length -eq 0) { diff --git a/src/functions/private/Read-YamlStreamCore.ps1 b/src/functions/private/Read-YamlStreamCore.ps1 index 5c61fb9..549aaa8 100644 --- a/src/functions/private/Read-YamlStreamCore.ps1 +++ b/src/functions/private/Read-YamlStreamCore.ps1 @@ -45,13 +45,13 @@ function Read-YamlStreamCore { -MaxAliases $MaxAliases -MaxScalarLength $MaxScalarLength ` -MaxTagLength $MaxTagLength -MaxTotalTagLength $MaxTotalTagLength ` -MaxNumericLength $MaxNumericLength - $text = $context.Text $lines = $context.Lines $lineStarts = $context.LineStarts $documents = [System.Collections.Generic.List[object]]::new() $implicitDocumentSeen = $false while ($context.LineIndex -lt $lines.Count) { + Read-YamlDocumentByteOrderMark -Context $context Skip-YamlBlockTrivia -Context $context if ($context.LineIndex -ge $lines.Count) { break @@ -76,7 +76,8 @@ function Read-YamlStreamCore { if ($context.LineIndex -ge $lines.Count) { if ($directiveSeen) { - $mark = New-YamlMark -Index $text.Length -Line ([Math]::Max(0, $lines.Count - 1)) -Column 0 + $mark = New-YamlMark -Index $context.Text.Length ` + -Line ([Math]::Max(0, $lines.Count - 1)) -Column 0 throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlDirectiveWithoutDocument' -Message ( 'YAML directives must be followed by an explicit document start marker.' )) @@ -113,7 +114,8 @@ function Read-YamlStreamCore { $context.LineIndex++ Skip-YamlBlockTrivia -Context $context if ($context.LineIndex -ge $lines.Count -or - $lines[$context.LineIndex] -match '^(?:---|\.\.\.)(?:[ \t]|$)') { + $lines[$context.LineIndex] -match '^(?:---|\.\.\.)(?:[ \t]|$)' -or + (Test-YamlDocumentByteOrderMark -Context $context -RequireDocumentStart)) { $mark = New-YamlMark -Index ($lineStarts[$markerLine] + 3) -Line $markerLine -Column 3 $document = New-YamlEmptyScalar -Context $context -Depth 1 -Mark $mark } else { @@ -162,6 +164,8 @@ function Read-YamlStreamCore { } $documents.Add($document) + Skip-YamlBlockTrivia -Context $context + Read-YamlDocumentByteOrderMark -Context $context -RequireDocumentStart Skip-YamlBlockTrivia -Context $context $explicitEnd = $false if ($context.LineIndex -lt $lines.Count -and diff --git a/src/functions/private/Test-YamlDocumentByteOrderMark.ps1 b/src/functions/private/Test-YamlDocumentByteOrderMark.ps1 new file mode 100644 index 0000000..ad1de12 --- /dev/null +++ b/src/functions/private/Test-YamlDocumentByteOrderMark.ps1 @@ -0,0 +1,25 @@ +function Test-YamlDocumentByteOrderMark { + <# + .SYNOPSIS + Tests for a byte order mark at the current document boundary. + #> + [CmdletBinding()] + [OutputType([bool])] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Context, + + [switch] $RequireDocumentStart + ) + + if ($Context.LineIndex -ge $Context.Lines.Count -or + -not $Context.Lines[$Context.LineIndex].StartsWith( + [string] [char] 0xFEFF, + [System.StringComparison]::Ordinal + )) { + return $false + } + return Test-YamlDocumentPrefix -Text $Context.Text ` + -Index ($Context.LineStarts[$Context.LineIndex] + 1) ` + -RequireDocumentStart:$RequireDocumentStart +} diff --git a/src/functions/private/Test-YamlDocumentPrefix.ps1 b/src/functions/private/Test-YamlDocumentPrefix.ps1 index d15adf5..e944f6f 100644 --- a/src/functions/private/Test-YamlDocumentPrefix.ps1 +++ b/src/functions/private/Test-YamlDocumentPrefix.ps1 @@ -11,7 +11,9 @@ function Test-YamlDocumentPrefix { [Parameter(Mandatory)] [ValidateRange(0, 2147483647)] - [int] $Index + [int] $Index, + + [switch] $RequireDocumentStart ) while ($Index -lt $Text.Length) { @@ -29,8 +31,13 @@ function Test-YamlDocumentPrefix { $Index = $lineEnd + 1 continue } - if ($line.StartsWith('%', [System.StringComparison]::Ordinal)) { - return $true + if ($line.StartsWith('%', [System.StringComparison]::Ordinal) -and + -not $RequireDocumentStart) { + if ($lineEnd -ge $Text.Length) { + return $false + } + $Index = $lineEnd + 1 + continue } if (-not $line.StartsWith('---', [System.StringComparison]::Ordinal)) { return $false diff --git a/tests/ConvertFrom-Yaml.Tests.ps1 b/tests/ConvertFrom-Yaml.Tests.ps1 index 6b9bb49..c113fff 100644 --- a/tests/ConvertFrom-Yaml.Tests.ps1 +++ b/tests/ConvertFrom-Yaml.Tests.ps1 @@ -207,14 +207,22 @@ folded: > "${bom}%YAML 1.2`n---`none`n...`n${bom}# prefix comment`n`n---`ntwo" | ConvertFrom-Yaml ) + $implicitBoundaryDocuments = @( + "---`none`n${bom}# prefix comment`n---`ntwo" | ConvertFrom-Yaml + ) $documents | Should -Be @('one', 'two') + $implicitBoundaryDocuments | Should -Be @('one', 'two') ("${bom}---`nvalue" | ConvertFrom-Yaml) | Should -Be 'value' ("foo${bom}bar" | Test-Yaml) | Should -BeFalse ("---`n${bom}value" | Test-Yaml) | Should -BeFalse ("---`none`n...`n${bom}# comment`ntwo" | Test-Yaml) | Should -BeFalse ('"foo' + $bom + 'bar"' | ConvertFrom-Yaml) | Should -Be "foo${bom}bar" ("'foo${bom}bar'" | ConvertFrom-Yaml) | Should -Be "foo${bom}bar" + ("`"a`n${bom}%foo`nb`"" | ConvertFrom-Yaml) | + Should -Be "a ${bom}%foo b" + ("|-`n${bom}%foo" | Test-Yaml) | Should -BeFalse + ("%FOO before${bom}after`n---`nvalue" | Test-Yaml) | Should -BeFalse } } From afa5bc90d8c3210b5e3f83e76a37f619fc2d8ea6 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 24 Jul 2026 02:19:45 +0200 Subject: [PATCH 18/24] Correct YAML mapping grammar edges Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../private/Find-YamlMappingColon.ps1 | 7 +++- .../private/Get-YamlImplicitKeyLength.ps1 | 13 +++++-- src/functions/private/Read-YamlBlockKey.ps1 | 2 +- .../private/Read-YamlBlockMapping.ps1 | 34 +++++++++++++++++-- .../private/Read-YamlDirectiveBlock.ps1 | 2 +- src/functions/private/Read-YamlFlowNode.ps1 | 25 ++++++-------- src/functions/private/Test-YamlIndicator.ps1 | 5 +-- tests/ConvertFrom-Yaml.Tests.ps1 | 16 +++++++++ tests/Test-Yaml.Tests.ps1 | 30 +++++++++++----- 9 files changed, 99 insertions(+), 35 deletions(-) diff --git a/src/functions/private/Find-YamlMappingColon.ps1 b/src/functions/private/Find-YamlMappingColon.ps1 index 9f2fc66..ada6943 100644 --- a/src/functions/private/Find-YamlMappingColon.ps1 +++ b/src/functions/private/Find-YamlMappingColon.ps1 @@ -50,7 +50,8 @@ function Find-YamlMappingColon { while ($index -lt $Text.Length -and -not (Test-YamlWhiteSpace -Character $Text[$index]) -and $Text[$index] -notin @(',', '[', ']', '{', '}')) { - if (Test-YamlMappingValueIndicator -Text $Text -Index $index) { + if ($index + 1 -lt $Text.Length -and + (Test-YamlMappingValueIndicator -Text $Text -Index $index)) { $anchorColonCandidate = $index } $index++ @@ -63,6 +64,10 @@ function Find-YamlMappingColon { while ($index -lt $Text.Length -and -not (Test-YamlWhiteSpace -Character $Text[$index]) -and $Text[$index] -notin @(',', '[', ']', '{', '}')) { + if ($index + 1 -lt $Text.Length -and + (Test-YamlMappingValueIndicator -Text $Text -Index $index)) { + $anchorColonCandidate = $index + } $index++ } $index-- diff --git a/src/functions/private/Get-YamlImplicitKeyLength.ps1 b/src/functions/private/Get-YamlImplicitKeyLength.ps1 index 63bee94..37c6c75 100644 --- a/src/functions/private/Get-YamlImplicitKeyLength.ps1 +++ b/src/functions/private/Get-YamlImplicitKeyLength.ps1 @@ -10,9 +10,18 @@ function Get-YamlImplicitKeyLength { [pscustomobject] $Node, [Parameter(Mandatory)] - [pscustomobject] $Context + [pscustomobject] $Context, + + [Parameter()] + [ValidateRange(0, 2147483647)] + [int] $EndIndex ) - $sourceLength = [Math]::Max(0, $Node.End.Index - $Node.Start.Index) + $sourceEnd = if ($PSBoundParameters.ContainsKey('EndIndex')) { + $EndIndex + } else { + $Node.End.Index + } + $sourceLength = [Math]::Max(0, $sourceEnd - $Node.Start.Index) return Get-YamlRuneCount -Text $Context.Text.Substring($Node.Start.Index, $sourceLength) } diff --git a/src/functions/private/Read-YamlBlockKey.ps1 b/src/functions/private/Read-YamlBlockKey.ps1 index 9b53d7d..2c0a208 100644 --- a/src/functions/private/Read-YamlBlockKey.ps1 +++ b/src/functions/private/Read-YamlBlockKey.ps1 @@ -48,7 +48,7 @@ function Read-YamlBlockKey { } $node = Read-YamlFlowNode -Cursor $cursor -Context $Context -Depth $Depth ` -PendingTag $properties.Tag -PendingUnknownTag $properties.HasUnknownTag ` - -PendingAnchor $properties.Anchor + -PendingAnchor $properties.Anchor -InImplicitKey $expectedEnd = $Context.LineStarts[$Line] + $Column + $Text.Length while ($cursor.Index -lt $expectedEnd -and $Context.Text[$cursor.Index] -in @(' ', "`t")) { Move-YamlCursor -Cursor $cursor -Context $Context diff --git a/src/functions/private/Read-YamlBlockMapping.ps1 b/src/functions/private/Read-YamlBlockMapping.ps1 index 562ee11..2fce98b 100644 --- a/src/functions/private/Read-YamlBlockMapping.ps1 +++ b/src/functions/private/Read-YamlBlockMapping.ps1 @@ -79,8 +79,21 @@ function Read-YamlBlockMapping { if ($isExplicit) { $afterQuestion = $content.Substring(1) $leading = $afterQuestion.Length - $afterQuestion.TrimStart(' ', "`t").Length + $separator = $afterQuestion.Substring(0, $leading) $keyText = $afterQuestion.TrimStart(' ', "`t") $keyColumn = $contentColumn + 1 + $leading + if ($separator.IndexOf("`t", [System.StringComparison]::Ordinal) -ge 0 -and + ( + (Test-YamlIndicator -Text $keyText -Indicator '-') -or + (Test-YamlIndicator -Text $keyText -Indicator '?') -or + (Find-YamlMappingColon -Text $keyText -AllowAnchorFallback) -ge 0 + )) { + $mark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $contentColumn + 1) ` + -Line $lineNumber -Column ($contentColumn + 1) + throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidIndentation' -Message ( + 'A tab cannot separate a mapping indicator from a compact block collection.' + )) + } if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $keyText))) { $Context.LineIndex++ Skip-YamlBlockTrivia -Context $Context @@ -133,10 +146,25 @@ function Read-YamlBlockMapping { } if ($valueIndent -eq $Indent -and (Test-YamlIndicator -Text $valueContent -Indicator ':')) { - $valueText = $valueContent.Substring(1) - $leading = $valueText.Length - $valueText.TrimStart(' ', "`t").Length - $valueText = $valueText.TrimStart(' ', "`t") + $valueSource = $valueContent.Substring(1) + $leading = $valueSource.Length - $valueSource.TrimStart(' ', "`t").Length + $separator = $valueSource.Substring(0, $leading) + $valueText = $valueSource.TrimStart(' ', "`t") $valueColumn = $Indent + 1 + $leading + if ($separator.IndexOf("`t", [System.StringComparison]::Ordinal) -ge 0 -and + ( + (Test-YamlIndicator -Text $valueText -Indicator '-') -or + (Test-YamlIndicator -Text $valueText -Indicator '?') -or + (Find-YamlMappingColon -Text $valueText -AllowAnchorFallback) -ge 0 + )) { + $mark = New-YamlMark -Index ( + $Context.LineStarts[$Context.LineIndex] + $Indent + 1 + ) -Line $Context.LineIndex -Column ($Indent + 1) + throw (New-YamlException -Start $mark -End $mark ` + -ErrorId 'YamlInvalidIndentation' -Message ( + 'A tab cannot separate a mapping indicator from a compact block collection.' + )) + } if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $valueText))) { $valueLineNumber = $Context.LineIndex $Context.LineIndex++ diff --git a/src/functions/private/Read-YamlDirectiveBlock.ps1 b/src/functions/private/Read-YamlDirectiveBlock.ps1 index d616eea..5031772 100644 --- a/src/functions/private/Read-YamlDirectiveBlock.ps1 +++ b/src/functions/private/Read-YamlDirectiveBlock.ps1 @@ -43,7 +43,7 @@ function Read-YamlDirectiveBlock { $yamlDirectiveSeen = $true } elseif ($directive -cmatch '^%TAG(?:[ \t]|$)') { if ($directive -cnotmatch ( - '^%TAG[ \t]+(!|!!|![0-9A-Za-z-]+!)[ \t]+([^ \t#]+)(?:[ \t]+#.*)?$' + '^%TAG[ \t]+(!|!!|![0-9A-Za-z-]+!)[ \t]+([^ \t]+)(?:[ \t]+#.*)?$' )) { throw (New-YamlException -Start $mark -End $mark ` -ErrorId 'YamlInvalidDirective' -Message ( diff --git a/src/functions/private/Read-YamlFlowNode.ps1 b/src/functions/private/Read-YamlFlowNode.ps1 index f1de657..4aa6e4b 100644 --- a/src/functions/private/Read-YamlFlowNode.ps1 +++ b/src/functions/private/Read-YamlFlowNode.ps1 @@ -24,7 +24,9 @@ function Read-YamlFlowNode { [AllowEmptyString()] [string] $PendingAnchor = '', - [switch] $InFlowCollection + [switch] $InFlowCollection, + + [switch] $InImplicitKey ) Skip-YamlFlowTrivia -Cursor $Cursor -Context $Context @@ -85,8 +87,9 @@ function Read-YamlFlowNode { -not (Test-YamlWhiteSpace -Character $Context.Text[$Cursor.Index]) -and $Context.Text[$Cursor.Index] -cne "`n" -and $Context.Text[$Cursor.Index] -notin @(',', '[', ']', '{', '}') -and - -not ($InFlowCollection -and - (Test-YamlMappingValueIndicator -Text $Context.Text -Index $Cursor.Index -Flow))) { + -not (($InFlowCollection -or $InImplicitKey) -and + (Test-YamlMappingValueIndicator -Text $Context.Text ` + -Index $Cursor.Index -Flow:$InFlowCollection))) { Move-YamlCursor -Cursor $Cursor -Context $Context } $anchor = $Context.Text.Substring($anchorStart, $Cursor.Index - $anchorStart) @@ -133,8 +136,9 @@ function Read-YamlFlowNode { -not (Test-YamlWhiteSpace -Character $Context.Text[$Cursor.Index]) -and $Context.Text[$Cursor.Index] -cne "`n" -and $Context.Text[$Cursor.Index] -notin @(',', '[', ']', '{', '}') -and - -not ($InFlowCollection -and - (Test-YamlMappingValueIndicator -Text $Context.Text -Index $Cursor.Index -Flow))) { + -not (($InFlowCollection -or $InImplicitKey) -and + (Test-YamlMappingValueIndicator -Text $Context.Text ` + -Index $Cursor.Index -Flow:$InFlowCollection))) { Move-YamlCursor -Cursor $Cursor -Context $Context } $alias = $Context.Text.Substring($aliasStart, $Cursor.Index - $aliasStart) @@ -192,7 +196,8 @@ function Read-YamlFlowNode { if ($Cursor.Index -lt $Context.Text.Length -and $Context.Text[$Cursor.Index] -eq ':') { if (-not $explicitPair -and ( $Cursor.Line -ne $itemStartLine -or - (Get-YamlImplicitKeyLength -Node $item -Context $Context) -gt 1024 + (Get-YamlImplicitKeyLength -Node $item -Context $Context ` + -EndIndex $Cursor.Index) -gt 1024 )) { $mark = New-YamlMark -Index $Cursor.Index -Line $Cursor.Line -Column $Cursor.Column throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidImplicitKey' -Message ( @@ -271,14 +276,6 @@ function Read-YamlFlowNode { } } Skip-YamlFlowTrivia -Cursor $Cursor -Context $Context - if (-not $explicit -and ( - (Get-YamlImplicitKeyLength -Node $key -Context $Context) -gt 1024 - )) { - $mark = New-YamlMark -Index $Cursor.Index -Line $Cursor.Line -Column $Cursor.Column - throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidImplicitKey' -Message ( - 'An implicit mapping key must fit on one line and cannot exceed 1024 Unicode scalar values.' - )) - } if ($Cursor.Index -ge $Context.Text.Length -or $Context.Text[$Cursor.Index] -ne ':') { if (-not $explicit) { if ($Cursor.Index -lt $Context.Text.Length -and diff --git a/src/functions/private/Test-YamlIndicator.ps1 b/src/functions/private/Test-YamlIndicator.ps1 index e264735..4e2de3c 100644 --- a/src/functions/private/Test-YamlIndicator.ps1 +++ b/src/functions/private/Test-YamlIndicator.ps1 @@ -21,8 +21,5 @@ function Test-YamlIndicator { if ($Text.Length -eq 1) { return $true } - if ($Indicator.Equals('-', [System.StringComparison]::Ordinal)) { - return Test-YamlWhiteSpace -Character $Text[1] - } - return $Text[1].Equals([char] ' ') + return Test-YamlWhiteSpace -Character $Text[1] } diff --git a/tests/ConvertFrom-Yaml.Tests.ps1 b/tests/ConvertFrom-Yaml.Tests.ps1 index c113fff..17e8da4 100644 --- a/tests/ConvertFrom-Yaml.Tests.ps1 +++ b/tests/ConvertFrom-Yaml.Tests.ps1 @@ -181,6 +181,15 @@ folded: > $sequencePair.Count | Should -Be 1 $sequencePair[0].foo | Should -Be @('bar') } + + It 'accepts tabs after explicit block mapping indicators' { + $tabAfterQuestion = "?`tkey`n: value" | ConvertFrom-Yaml -AsHashtable + $tabAfterColon = "? key`n:`tvalue" | ConvertFrom-Yaml -AsHashtable + + $tabAfterQuestion['key'] | Should -Be 'value' + $tabAfterColon['key'] | Should -Be 'value' + $tabAfterColon.Count | Should -Be 1 + } } Context 'Streams and pipeline input' { @@ -231,11 +240,18 @@ folded: > $emptyKey = '&a: value' | ConvertFrom-Yaml -AsHashtable $aliasedKey = '{anchor: &a foo, *a: value}' | ConvertFrom-Yaml -AsHashtable + $blockAliasedKey = "source: &a key`n*a: value" | + ConvertFrom-Yaml -AsHashtable + $colonAliasName = "&a: key: &a value`nfoo:`n *a:" | + ConvertFrom-Yaml -AsHashtable $emptyKey.Count | Should -Be 1 $emptyKey[[System.DBNull]::Value] | Should -Be 'value' @($aliasedKey.Keys) | Should -Be @('anchor', 'foo') @($aliasedKey.Values) | Should -Be @('foo', 'value') + @($blockAliasedKey.Keys) | Should -Be @('source', 'key') + @($blockAliasedKey.Values) | Should -Be @('key', 'value') + $colonAliasName['foo'] | Should -Be 'key' } It 'constructs explicit standard scalar tags safely' { diff --git a/tests/Test-Yaml.Tests.ps1 b/tests/Test-Yaml.Tests.ps1 index 7a34d35..2793a6e 100644 --- a/tests/Test-Yaml.Tests.ps1 +++ b/tests/Test-Yaml.Tests.ps1 @@ -37,7 +37,7 @@ Describe 'Test-Yaml' { ("a: &a value`nb: !foo *a" | Test-Yaml) | Should -BeFalse } - It 'enforces implicit-key line and Unicode scalar limits' { + It 'enforces block and flow-sequence implicit-key limits' { $emoji = [char]::ConvertFromUtf32(0x1F600) ("['multi`n line': value]" | Test-Yaml) | Should -BeFalse @@ -53,23 +53,19 @@ Describe 'Test-Yaml' { @{ Key = '!local ' + ('k' * 1017); Valid = $true } @{ Key = '!local ' + ('k' * 1018); Valid = $false } )) { - foreach ($yaml in @( - "[$($case.Key)`: value]", - "{$($case.Key)`: value}" - )) { - ($yaml | Test-Yaml) | Should -Be $case.Valid - } + ("[$($case.Key)`: value]" | Test-Yaml) | Should -Be $case.Valid } foreach ($length in @(513, 1024, 1025)) { $key = $emoji * $length - ("{$key`: value}" | Test-Yaml) | + ("[$key`: value]" | Test-Yaml) | Should -Be ($length -le 1024) } + ("['$('k' * 1021)' : value]" | Test-Yaml) | Should -BeTrue + ("['$('k' * 1022)' : value]" | Test-Yaml) | Should -BeFalse ((('k' * 1023) + ' : value') | Test-Yaml) | Should -BeTrue ((('k' * 1024) + ' : value') | Test-Yaml) | Should -BeFalse - } It 'resolves non-specific tags before comparing representation keys' { @@ -84,9 +80,18 @@ Describe 'Test-Yaml' { ("{multi`n line: value}" | Test-Yaml) | Should -BeTrue } + It 'does not apply implicit-key limits to ordinary flow mappings' { + $emoji = [char]::ConvertFromUtf32(0x1F600) + + (('{' + ('k' * 1025) + ': value}') | Test-Yaml) | Should -BeTrue + (('{' + ($emoji * 1025) + ': value}') | Test-Yaml) | Should -BeTrue + } + It 'validates tag directives, tag tokens, and alias properties' { ("%TAG !! tag:example.com,2000:app/`n---`n!!int value" | Test-Yaml) | Should -BeTrue + ("%TAG !e! tag:example.com,2000:app/#fragment`n---`n!e!foo value" | + Test-Yaml) | Should -BeTrue ("%FOO-BAR baz`n---`nvalue" | Test-Yaml) | Should -BeTrue ("%FOO:B alpha-beta p#q`n---`nvalue" | Test-Yaml) | Should -BeTrue ("%TAG ! tag:first/`n%TAG ! tag:second/`n---`n!value data" | Test-Yaml) | @@ -108,6 +113,13 @@ Describe 'Test-Yaml' { ("key:`n ..." | Test-Yaml) | Should -BeTrue } + It 'rejects tabs that indent compact collections after mapping indicators' { + ("?`t-" | Test-Yaml) | Should -BeFalse + ("? -`n:`t-" | Test-Yaml) | Should -BeFalse + ("?`tkey:" | Test-Yaml) | Should -BeFalse + ("? key:`n:`tkey:" | Test-Yaml) | Should -BeFalse + } + It 'returns false for duplicate mapping keys' { ("key: one`nkey: two" | Test-Yaml) | Should -BeFalse ("1: one`n01: two" | Test-Yaml) | Should -BeFalse From 6a9d2c5e0b931a4240fa5c8b4411029df39b0ed8 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 24 Jul 2026 02:25:05 +0200 Subject: [PATCH 19/24] Harden YAML document prefixes Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../private/Get-YamlContentWithoutComment.ps1 | 8 ++++- .../private/Read-YamlBlockMapping.ps1 | 29 ++++++++++++++--- src/functions/private/Read-YamlBlockNode.ps1 | 4 ++- .../private/Read-YamlBlockScalar.ps1 | 4 +++ .../private/Read-YamlBlockSequence.ps1 | 17 ++++++++-- .../private/Read-YamlPlainScalar.ps1 | 7 ++-- src/functions/private/Read-YamlStreamCore.ps1 | 32 +++++++++++-------- .../private/Skip-YamlBlockTrivia.ps1 | 10 +++++- .../private/Skip-YamlDocumentPrefix.ps1 | 28 ++++++++++++++++ src/functions/private/Skip-YamlFlowTrivia.ps1 | 25 ++++++++++++++- .../private/Test-YamlDocumentPrefix.ps1 | 10 ++++-- tests/ConvertFrom-Yaml.Tests.ps1 | 16 ++++++++++ 12 files changed, 162 insertions(+), 28 deletions(-) create mode 100644 src/functions/private/Skip-YamlDocumentPrefix.ps1 diff --git a/src/functions/private/Get-YamlContentWithoutComment.ps1 b/src/functions/private/Get-YamlContentWithoutComment.ps1 index 9f39bb4..64d1e25 100644 --- a/src/functions/private/Get-YamlContentWithoutComment.ps1 +++ b/src/functions/private/Get-YamlContentWithoutComment.ps1 @@ -8,11 +8,17 @@ function Get-YamlContentWithoutComment { param ( [Parameter(Mandatory)] [AllowEmptyString()] - [string] $Text + [string] $Text, + + [Parameter(Mandatory)] + [pscustomobject] $Mark ) $comment = Find-YamlCommentStart -Text $Text if ($comment -ge 0) { + $commentMark = New-YamlMark -Index ($Mark.Index + $comment) -Line $Mark.Line ` + -Column ($Mark.Column + $comment) + Assert-YamlNoByteOrderMark -Text $Text.Substring($comment) -Mark $commentMark return $Text.Substring(0, $comment).TrimEnd(' ', "`t") } return $Text.TrimEnd(' ', "`t") diff --git a/src/functions/private/Read-YamlBlockMapping.ps1 b/src/functions/private/Read-YamlBlockMapping.ps1 index 2fce98b..3ef255e 100644 --- a/src/functions/private/Read-YamlBlockMapping.ps1 +++ b/src/functions/private/Read-YamlBlockMapping.ps1 @@ -53,7 +53,15 @@ function Read-YamlBlockMapping { (Test-YamlDocumentByteOrderMark -Context $Context -RequireDocumentStart)) { break } - if ($trimmed.Length -eq 0 -or $trimmed.StartsWith('#', [System.StringComparison]::Ordinal)) { + if ($trimmed.StartsWith('#', [System.StringComparison]::Ordinal)) { + $column = $line.Length - $trimmed.Length + $mark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $column) ` + -Line $lineNumber -Column $column + Assert-YamlNoByteOrderMark -Text $trimmed -Mark $mark + $Context.LineIndex++ + continue + } + if ($trimmed.Length -eq 0) { $Context.LineIndex++ continue } @@ -94,7 +102,11 @@ function Read-YamlBlockMapping { 'A tab cannot separate a mapping indicator from a compact block collection.' )) } - if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $keyText))) { + $keyMark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $keyColumn) ` + -Line $lineNumber -Column $keyColumn + if ([string]::IsNullOrEmpty(( + Get-YamlContentWithoutComment -Text $keyText -Mark $keyMark + ))) { $Context.LineIndex++ Skip-YamlBlockTrivia -Context $Context if ($Context.LineIndex -ge $Context.Lines.Count) { @@ -165,7 +177,12 @@ function Read-YamlBlockMapping { 'A tab cannot separate a mapping indicator from a compact block collection.' )) } - if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $valueText))) { + $valueMark = New-YamlMark -Index ( + $Context.LineStarts[$Context.LineIndex] + $valueColumn + ) -Line $Context.LineIndex -Column $valueColumn + if ([string]::IsNullOrEmpty(( + Get-YamlContentWithoutComment -Text $valueText -Mark $valueMark + ))) { $valueLineNumber = $Context.LineIndex $Context.LineIndex++ Skip-YamlBlockTrivia -Context $Context @@ -234,7 +251,11 @@ function Read-YamlBlockMapping { $leading = $valueSource.Length - $valueSource.TrimStart(' ', "`t").Length $valueText = $valueSource.TrimStart(' ', "`t") $valueColumn = $contentColumn + $valueStart + $leading - if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $valueText))) { + $valueMark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $valueColumn) ` + -Line $lineNumber -Column $valueColumn + if ([string]::IsNullOrEmpty(( + Get-YamlContentWithoutComment -Text $valueText -Mark $valueMark + ))) { $Context.LineIndex = $lineNumber + 1 Skip-YamlBlockTrivia -Context $Context if ($Context.LineIndex -lt $Context.Lines.Count) { diff --git a/src/functions/private/Read-YamlBlockNode.ps1 b/src/functions/private/Read-YamlBlockNode.ps1 index 38a24a5..e160baa 100644 --- a/src/functions/private/Read-YamlBlockNode.ps1 +++ b/src/functions/private/Read-YamlBlockNode.ps1 @@ -113,8 +113,10 @@ function Read-YamlBlockNode { $unknownTag = $properties.HasUnknownTag -or $PendingUnknownTag $anchor = if (-not [string]::IsNullOrEmpty($properties.Anchor)) { $properties.Anchor } else { $PendingAnchor } $restSource = $properties.Rest - $rest = Get-YamlContentWithoutComment -Text $restSource $contentColumn = $SegmentColumn + $properties.Consumed + $restMark = New-YamlMark -Index ($Context.LineStarts[$lineNumber] + $contentColumn) ` + -Line $lineNumber -Column $contentColumn + $rest = Get-YamlContentWithoutComment -Text $restSource -Mark $restMark if ([string]::IsNullOrEmpty($rest)) { $Context.LineIndex++ diff --git a/src/functions/private/Read-YamlBlockScalar.ps1 b/src/functions/private/Read-YamlBlockScalar.ps1 index b3d984d..970c0d3 100644 --- a/src/functions/private/Read-YamlBlockScalar.ps1 +++ b/src/functions/private/Read-YamlBlockScalar.ps1 @@ -31,6 +31,10 @@ function Read-YamlBlockScalar { [string] $Anchor = '' ) + $headerMark = New-YamlMark -Index ( + $Context.LineStarts[$Context.LineIndex] + $HeaderColumn + ) -Line $Context.LineIndex -Column $HeaderColumn + Assert-YamlNoByteOrderMark -Text $Header -Mark $headerMark if ($Header -notmatch ( '^([|>])(?:(?:([1-9])([+-])?)|(?:([+-])([1-9])?))?' + '(?:[ \t]+#.*|[ \t]*)$' diff --git a/src/functions/private/Read-YamlBlockSequence.ps1 b/src/functions/private/Read-YamlBlockSequence.ps1 index cb8aba7..29ae86d 100644 --- a/src/functions/private/Read-YamlBlockSequence.ps1 +++ b/src/functions/private/Read-YamlBlockSequence.ps1 @@ -51,7 +51,15 @@ function Read-YamlBlockSequence { (Test-YamlDocumentByteOrderMark -Context $Context -RequireDocumentStart)) { break } - if ($trimmed.Length -eq 0 -or $trimmed.StartsWith('#', [System.StringComparison]::Ordinal)) { + if ($trimmed.StartsWith('#', [System.StringComparison]::Ordinal)) { + $column = $line.Length - $trimmed.Length + $mark = New-YamlMark -Index ($Context.LineStarts[$Context.LineIndex] + $column) ` + -Line $Context.LineIndex -Column $column + Assert-YamlNoByteOrderMark -Text $trimmed -Mark $mark + $Context.LineIndex++ + continue + } + if ($trimmed.Length -eq 0) { $Context.LineIndex++ continue } @@ -70,7 +78,12 @@ function Read-YamlBlockSequence { ) } - if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $rest))) { + $restMark = New-YamlMark -Index ( + $Context.LineStarts[$Context.LineIndex] + $itemColumn + ) -Line $Context.LineIndex -Column $itemColumn + if ([string]::IsNullOrEmpty(( + Get-YamlContentWithoutComment -Text $rest -Mark $restMark + ))) { $line = $Context.LineIndex $Context.LineIndex++ Skip-YamlBlockTrivia -Context $Context diff --git a/src/functions/private/Read-YamlPlainScalar.ps1 b/src/functions/private/Read-YamlPlainScalar.ps1 index d804442..ff4507c 100644 --- a/src/functions/private/Read-YamlPlainScalar.ps1 +++ b/src/functions/private/Read-YamlPlainScalar.ps1 @@ -36,7 +36,8 @@ function Read-YamlPlainScalar { -Column $FirstColumn $parts = [System.Collections.Generic.List[object]]::new() $firstComment = Find-YamlCommentStart -Text $FirstText - $firstValue = (Get-YamlContentWithoutComment -Text $FirstText).Trim(' ', "`t") + $firstValue = Get-YamlContentWithoutComment -Text $FirstText -Mark $start + $firstValue = $firstValue.Trim(' ', "`t") Assert-YamlNoByteOrderMark -Text $firstValue -Mark $start $firstCharacter = if ($firstValue.Length -gt 0) { $firstValue[0] } else { [char] 0 } $forbiddenFirst = $firstCharacter -in @( @@ -88,7 +89,9 @@ function Read-YamlPlainScalar { } $sourceContent = $line.Substring($indent) $comment = Find-YamlCommentStart -Text $sourceContent - $content = Get-YamlContentWithoutComment -Text $sourceContent + $contentMark = New-YamlMark -Index ($Context.LineStarts[$Context.LineIndex] + $indent) ` + -Line $Context.LineIndex -Column $indent + $content = Get-YamlContentWithoutComment -Text $sourceContent -Mark $contentMark if ((Find-YamlMappingColon -Text $content) -ge 0) { break } diff --git a/src/functions/private/Read-YamlStreamCore.ps1 b/src/functions/private/Read-YamlStreamCore.ps1 index 549aaa8..3e8087b 100644 --- a/src/functions/private/Read-YamlStreamCore.ps1 +++ b/src/functions/private/Read-YamlStreamCore.ps1 @@ -51,16 +51,16 @@ function Read-YamlStreamCore { $implicitDocumentSeen = $false while ($context.LineIndex -lt $lines.Count) { - Read-YamlDocumentByteOrderMark -Context $context - Skip-YamlBlockTrivia -Context $context + Skip-YamlDocumentPrefix -Context $context if ($context.LineIndex -ge $lines.Count) { break } if ($lines[$context.LineIndex] -match '^\.\.\.(?:[ \t]|$)') { - $suffix = Get-YamlContentWithoutComment -Text $lines[$context.LineIndex].Substring(3) + $mark = New-YamlMark -Index ($lineStarts[$context.LineIndex] + 3) ` + -Line $context.LineIndex -Column 3 + $suffix = Get-YamlContentWithoutComment ` + -Text $lines[$context.LineIndex].Substring(3) -Mark $mark if ($suffix.Trim(' ', "`t").Length -gt 0) { - $mark = New-YamlMark -Index ($lineStarts[$context.LineIndex] + 3) ` - -Line $context.LineIndex -Column 3 throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidDocumentEnd' -Message ( 'Unexpected content follows the document end marker.' )) @@ -109,7 +109,12 @@ function Read-YamlStreamCore { $leading = $afterMarker.Length - $afterMarker.TrimStart(' ', "`t").Length $segment = $afterMarker.TrimStart(' ', "`t") $segmentColumn = 3 + $leading - if ([string]::IsNullOrEmpty((Get-YamlContentWithoutComment -Text $segment))) { + $segmentMark = New-YamlMark -Index ( + $lineStarts[$context.LineIndex] + $segmentColumn + ) -Line $context.LineIndex -Column $segmentColumn + if ([string]::IsNullOrEmpty(( + Get-YamlContentWithoutComment -Text $segment -Mark $segmentMark + ))) { $markerLine = $context.LineIndex $context.LineIndex++ Skip-YamlBlockTrivia -Context $context @@ -164,24 +169,23 @@ function Read-YamlStreamCore { } $documents.Add($document) - Skip-YamlBlockTrivia -Context $context - Read-YamlDocumentByteOrderMark -Context $context -RequireDocumentStart - Skip-YamlBlockTrivia -Context $context + Skip-YamlDocumentPrefix -Context $context -RequireDocumentStart $explicitEnd = $false if ($context.LineIndex -lt $lines.Count -and $lines[$context.LineIndex] -match '^\.\.\.(?:[ \t]|$)') { $explicitEnd = $true $endLine = $lines[$context.LineIndex] - if ((Get-YamlContentWithoutComment -Text $endLine.Substring(3)). + $endMark = New-YamlMark -Index ($lineStarts[$context.LineIndex] + 3) ` + -Line $context.LineIndex -Column 3 + if ((Get-YamlContentWithoutComment -Text $endLine.Substring(3) -Mark $endMark). Trim(' ', "`t").Length -gt 0) { - $mark = New-YamlMark -Index ($lineStarts[$context.LineIndex] + 3) ` - -Line $context.LineIndex -Column 3 - throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidDocumentEnd' -Message ( + throw (New-YamlException -Start $endMark -End $endMark ` + -ErrorId 'YamlInvalidDocumentEnd' -Message ( 'Unexpected content follows the document end marker.' )) } $context.LineIndex++ - Skip-YamlBlockTrivia -Context $context + Skip-YamlDocumentPrefix -Context $context $implicitDocumentSeen = $false } if (-not $explicitEnd -and $context.LineIndex -lt $lines.Count -and diff --git a/src/functions/private/Skip-YamlBlockTrivia.ps1 b/src/functions/private/Skip-YamlBlockTrivia.ps1 index d1603e9..536843f 100644 --- a/src/functions/private/Skip-YamlBlockTrivia.ps1 +++ b/src/functions/private/Skip-YamlBlockTrivia.ps1 @@ -16,7 +16,15 @@ function Skip-YamlBlockTrivia { while ($Context.LineIndex -lt $Context.Lines.Count) { $line = $Context.Lines[$Context.LineIndex] $trimmed = $line.TrimStart(' ', "`t") - if ($trimmed.Length -eq 0 -or $trimmed.StartsWith('#', [System.StringComparison]::Ordinal)) { + if ($trimmed.StartsWith('#', [System.StringComparison]::Ordinal)) { + $column = $line.Length - $trimmed.Length + $mark = New-YamlMark -Index ($Context.LineStarts[$Context.LineIndex] + $column) ` + -Line $Context.LineIndex -Column $column + Assert-YamlNoByteOrderMark -Text $trimmed -Mark $mark + $Context.LineIndex++ + continue + } + if ($trimmed.Length -eq 0) { $Context.LineIndex++ continue } diff --git a/src/functions/private/Skip-YamlDocumentPrefix.ps1 b/src/functions/private/Skip-YamlDocumentPrefix.ps1 new file mode 100644 index 0000000..481a881 --- /dev/null +++ b/src/functions/private/Skip-YamlDocumentPrefix.ps1 @@ -0,0 +1,28 @@ +function Skip-YamlDocumentPrefix { + <# + .SYNOPSIS + Advances across repeated YAML document prefixes. + #> + [Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSUseShouldProcessForStateChangingFunctions', '', + Justification = 'Advances an in-memory parser context.' + )] + [CmdletBinding()] + param ( + [Parameter(Mandatory)] + [pscustomobject] $Context, + + [switch] $RequireDocumentStart + ) + + do { + $lineIndex = $Context.LineIndex + $textLength = $Context.Text.Length + Read-YamlDocumentByteOrderMark -Context $Context ` + -RequireDocumentStart:$RequireDocumentStart + Skip-YamlBlockTrivia -Context $Context + } while ( + $Context.LineIndex -ne $lineIndex -or + $Context.Text.Length -ne $textLength + ) +} diff --git a/src/functions/private/Skip-YamlFlowTrivia.ps1 b/src/functions/private/Skip-YamlFlowTrivia.ps1 index c88ab9c..5ddc5f4 100644 --- a/src/functions/private/Skip-YamlFlowTrivia.ps1 +++ b/src/functions/private/Skip-YamlFlowTrivia.ps1 @@ -23,13 +23,24 @@ function Skip-YamlFlowTrivia { continue } if ($character -eq '#') { + $mark = New-YamlMark -Index $Cursor.Index -Line $Cursor.Line -Column $Cursor.Column if ($Cursor.Index -gt 0 -and $Context.Text[$Cursor.Index - 1] -notin @(' ', "`t", "`n")) { - $mark = New-YamlMark -Index $Cursor.Index -Line $Cursor.Line -Column $Cursor.Column throw (New-YamlException -Start $mark -End $mark -ErrorId 'YamlInvalidComment' -Message ( 'A YAML comment must be separated from preceding content.' )) } + $commentEnd = $Context.Text.IndexOf( + "`n", + $Cursor.Index, + [System.StringComparison]::Ordinal + ) + if ($commentEnd -lt 0) { + $commentEnd = $Context.Text.Length + } + Assert-YamlNoByteOrderMark ` + -Text $Context.Text.Substring($Cursor.Index, $commentEnd - $Cursor.Index) ` + -Mark $mark while ($Cursor.Index -lt $Context.Text.Length -and $Context.Text[$Cursor.Index] -ne "`n") { Move-YamlCursor -Cursor $Cursor -Context $Context } @@ -56,6 +67,18 @@ function Skip-YamlFlowTrivia { return } if ($Context.Text[$Cursor.Index] -eq '#') { + $mark = New-YamlMark -Index $Cursor.Index -Line $Cursor.Line -Column $Cursor.Column + $commentEnd = $Context.Text.IndexOf( + "`n", + $Cursor.Index, + [System.StringComparison]::Ordinal + ) + if ($commentEnd -lt 0) { + $commentEnd = $Context.Text.Length + } + Assert-YamlNoByteOrderMark ` + -Text $Context.Text.Substring($Cursor.Index, $commentEnd - $Cursor.Index) ` + -Mark $mark while ($Cursor.Index -lt $Context.Text.Length -and $Context.Text[$Cursor.Index] -ne "`n") { Move-YamlCursor -Cursor $Cursor -Context $Context } diff --git a/src/functions/private/Test-YamlDocumentPrefix.ps1 b/src/functions/private/Test-YamlDocumentPrefix.ps1 index e944f6f..9354b2a 100644 --- a/src/functions/private/Test-YamlDocumentPrefix.ps1 +++ b/src/functions/private/Test-YamlDocumentPrefix.ps1 @@ -16,7 +16,12 @@ function Test-YamlDocumentPrefix { [switch] $RequireDocumentStart ) + $directiveSeen = $false while ($Index -lt $Text.Length) { + if ($Text[$Index] -ceq [char] 0xFEFF) { + $Index++ + continue + } $lineEnd = $Text.IndexOf("`n", $Index, [System.StringComparison]::Ordinal) if ($lineEnd -lt 0) { $lineEnd = $Text.Length @@ -26,13 +31,14 @@ function Test-YamlDocumentPrefix { if ($trimmed.Length -eq 0 -or $trimmed.StartsWith('#', [System.StringComparison]::Ordinal)) { if ($lineEnd -ge $Text.Length) { - return $false + return -not $directiveSeen } $Index = $lineEnd + 1 continue } if ($line.StartsWith('%', [System.StringComparison]::Ordinal) -and -not $RequireDocumentStart) { + $directiveSeen = $true if ($lineEnd -ge $Text.Length) { return $false } @@ -45,5 +51,5 @@ function Test-YamlDocumentPrefix { return $line.Length -eq 3 -or (Test-YamlWhiteSpace -Character $line[3]) } - return $false + return -not $directiveSeen } diff --git a/tests/ConvertFrom-Yaml.Tests.ps1 b/tests/ConvertFrom-Yaml.Tests.ps1 index 17e8da4..93add1f 100644 --- a/tests/ConvertFrom-Yaml.Tests.ps1 +++ b/tests/ConvertFrom-Yaml.Tests.ps1 @@ -219,9 +219,21 @@ folded: > $implicitBoundaryDocuments = @( "---`none`n${bom}# prefix comment`n---`ntwo" | ConvertFrom-Yaml ) + $commentThenBom = @( + "# first`n${bom}---`nvalue" | ConvertFrom-Yaml + ) + $alternatingPrefixes = @( + "# first`n${bom}# second`n${bom}---`nvalue" | ConvertFrom-Yaml + ) + $trailingPrefix = @( + "---`nvalue`n...`n${bom}# trailing prefix" | ConvertFrom-Yaml + ) $documents | Should -Be @('one', 'two') $implicitBoundaryDocuments | Should -Be @('one', 'two') + $commentThenBom | Should -Be @('value') + $alternatingPrefixes | Should -Be @('value') + $trailingPrefix | Should -Be @('value') ("${bom}---`nvalue" | ConvertFrom-Yaml) | Should -Be 'value' ("foo${bom}bar" | Test-Yaml) | Should -BeFalse ("---`n${bom}value" | Test-Yaml) | Should -BeFalse @@ -232,6 +244,10 @@ folded: > Should -Be "a ${bom}%foo b" ("|-`n${bom}%foo" | Test-Yaml) | Should -BeFalse ("%FOO before${bom}after`n---`nvalue" | Test-Yaml) | Should -BeFalse + ("value # a${bom}b" | Test-Yaml) | Should -BeFalse + ("# a${bom}b`nvalue" | Test-Yaml) | Should -BeFalse + ("[value # a${bom}b`n ]" | Test-Yaml) | Should -BeFalse + ("| # a${bom}b`n value" | Test-Yaml) | Should -BeFalse } } From 2d388d056ba7b5c8d151026bd45ac02d3edc0914 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 24 Jul 2026 02:33:56 +0200 Subject: [PATCH 20/24] Separate YAML conformance semantics Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/Conformance.Tests.ps1 | 70 ++++++- tests/tools/Invoke-YamlTestSuite.ps1 | 262 ++++++++++++++++++--------- 2 files changed, 239 insertions(+), 93 deletions(-) diff --git a/tests/Conformance.Tests.ps1 b/tests/Conformance.Tests.ps1 index 8168ca3..16810ec 100644 --- a/tests/Conformance.Tests.ps1 +++ b/tests/Conformance.Tests.ps1 @@ -168,7 +168,32 @@ Describe 'Released yaml-test-suite corpus accounting' { $secondOrder.Add('b', 2) $secondOrder.Add('a', 1) (ConvertTo-YamlSuiteCanonicalValue -Value $firstOrder) | - Should -Not -Be (ConvertTo-YamlSuiteCanonicalValue -Value $secondOrder) + Should -Be (ConvertTo-YamlSuiteCanonicalValue -Value $secondOrder) + + $orderedMappings = [System.Collections.Generic.HashSet[object]]::new( + [System.Collections.Generic.ReferenceEqualityComparer]::Instance + ) + [void] $orderedMappings.Add($firstOrder) + [void] $orderedMappings.Add($secondOrder) + (ConvertTo-YamlSuiteCanonicalValue -Value $firstOrder ` + -OrderedMappings $orderedMappings) | + Should -Not -Be ( + ConvertTo-YamlSuiteCanonicalValue -Value $secondOrder ` + -OrderedMappings $orderedMappings + ) + + $firstNestedKey = [System.Collections.Specialized.OrderedDictionary]::new() + $secondNestedKey = [System.Collections.Specialized.OrderedDictionary]::new() + $firstNestedKey.Add($firstOrder, 'value') + $secondNestedKey.Add($secondOrder, 'value') + (ConvertTo-YamlSuiteCanonicalValue -Value $firstNestedKey) | + Should -Be (ConvertTo-YamlSuiteCanonicalValue -Value $secondNestedKey) + (ConvertTo-YamlSuiteCanonicalValue -Value $firstNestedKey ` + -OrderedMappings $orderedMappings) | + Should -Not -Be ( + ConvertTo-YamlSuiteCanonicalValue -Value $secondNestedKey ` + -OrderedMappings $orderedMappings + ) (ConvertTo-YamlSuiteCanonicalValue -Value ([uri] 'https://example.com/one')) | Should -Not -Be ( @@ -233,6 +258,20 @@ ship-to: Should -Be @('OutYamlConstructionMismatch', 'OutYamlReferenceMismatch') } + It 'ignores source order for ordinary mapping out.yaml comparisons' { + $ordinarySuitePath = Join-Path $TestDrive 'ordinary-mapping-order' + $ordinaryCasePath = Join-Path $ordinarySuitePath 'ordinary-map' + $null = New-Item -Path $ordinaryCasePath -ItemType Directory -Force + "a: 1`nb: 2" | Set-Content -LiteralPath (Join-Path $ordinaryCasePath 'in.yaml') ` + -Encoding utf8NoBOM + "b: 2`na: 1" | Set-Content -LiteralPath (Join-Path $ordinaryCasePath 'out.yaml') ` + -Encoding utf8NoBOM + + $ordinaryResult = & $runnerPath -Path $ordinarySuitePath -CompareOutYaml + + $ordinaryResult.OutYamlResult | Should -Be 'Pass' + } + It 'accounts for out.yaml representation comparisons' { @($suiteResults | Where-Object OutYamlResult -EQ 'Pass').Count | Should -Be 241 @($suiteResults | Where-Object OutYamlResult -EQ 'PolicyDifference').Count | @@ -295,11 +334,27 @@ ship-to: Should -Be @('EmitYamlRepresentationMismatch') } + It 'does not run module emission while validating official fixtures' { + $independentSuitePath = Join-Path $TestDrive 'independent-emit-fixture' + $independentCasePath = Join-Path $independentSuitePath 'deep-fixture' + $null = New-Item -Path $independentCasePath -ItemType Directory -Force + $deepYaml = ('[' * 101) + 'value' + (']' * 101) + $deepYaml | Set-Content -LiteralPath (Join-Path $independentCasePath 'in.yaml') ` + -Encoding utf8NoBOM -NoNewline + $deepYaml | Set-Content -LiteralPath (Join-Path $independentCasePath 'emit.yaml') ` + -Encoding utf8NoBOM -NoNewline + + $independentResult = & $runnerPath -Path $independentSuitePath -CompareEmitYaml + + $independentResult.EmitYamlResult | Should -Be 'Pass' + $independentResult.SelfRoundTripResult | Should -Be 'NotApplicable' + } + It 'accounts honestly for general module self-round-trips' { @($suiteResults | Where-Object SelfRoundTripResult -EQ 'Pass').Count | - Should -Be 306 + Should -Be 305 @($suiteResults | Where-Object SelfRoundTripResult -EQ 'PolicyDifference').Count | - Should -Be 2 + Should -Be 3 @($suiteResults | Where-Object SelfRoundTripResult -EQ 'Fail').Count | Should -Be 0 @($suiteResults | Where-Object SelfRoundTripResult -EQ 'NotApplicable').Count | @@ -309,9 +364,12 @@ ship-to: Where-Object SelfRoundTripResult -EQ 'PolicyDifference' | Sort-Object Case ) - @($policyResults.Case) | Should -Be @('2JQS', 'X38W') - @($policyResults.SelfRoundTripReason | Select-Object -Unique) | - Should -Be @('RepresentationMappingKeyUniqueness') + @($policyResults.Case) | Should -Be @('2JQS', 'J7PZ', 'X38W') + @($policyResults.SelfRoundTripReason) | Should -Be @( + 'RepresentationMappingKeyUniqueness', + 'LegacyOrderedMapProjection', + 'RepresentationMappingKeyUniqueness' + ) } It 'keeps the previously failing multi-document JSON cases green' { diff --git a/tests/tools/Invoke-YamlTestSuite.ps1 b/tests/tools/Invoke-YamlTestSuite.ps1 index e30fb45..a0be294 100644 --- a/tests/tools/Invoke-YamlTestSuite.ps1 +++ b/tests/tools/Invoke-YamlTestSuite.ps1 @@ -120,7 +120,10 @@ function ConvertTo-YamlSuiteCanonicalValue { [AllowNull()] [object] $Value, - [switch] $SortMappings + [switch] $SortMappings, + + [AllowNull()] + [System.Collections.Generic.HashSet[object]] $OrderedMappings ) if ($null -eq $Value -or $Value -is [System.DBNull]) { @@ -182,9 +185,9 @@ function ConvertTo-YamlSuiteCanonicalValue { $entries = [System.Collections.Generic.List[string]]::new() foreach ($entry in $Value.GetEnumerator()) { $canonicalKey = ConvertTo-YamlSuiteCanonicalValue -Value $entry.Key ` - -SortMappings:$SortMappings + -SortMappings:$SortMappings -OrderedMappings $OrderedMappings $canonicalValue = ConvertTo-YamlSuiteCanonicalValue -Value $entry.Value ` - -SortMappings:$SortMappings + -SortMappings:$SortMappings -OrderedMappings $OrderedMappings $entries.Add(('{0}:{1}={2}:{3}' -f $canonicalKey.Length, $canonicalKey, @@ -192,10 +195,7 @@ function ConvertTo-YamlSuiteCanonicalValue { $canonicalValue )) } - $isOrdered = ( - $Value -is [System.Collections.Specialized.OrderedDictionary] -or - $Value.GetType().FullName -ceq 'System.Management.Automation.OrderedHashtable' - ) + $isOrdered = $null -ne $OrderedMappings -and $OrderedMappings.Contains($Value) if ($SortMappings -or -not $isOrdered) { $entries.Sort([System.StringComparer]::Ordinal) } @@ -205,7 +205,8 @@ function ConvertTo-YamlSuiteCanonicalValue { $items = [System.Collections.Generic.List[string]]::new() foreach ($item in $Value) { $items.Add(( - ConvertTo-YamlSuiteCanonicalValue -Value $item -SortMappings:$SortMappings + ConvertTo-YamlSuiteCanonicalValue -Value $item -SortMappings:$SortMappings ` + -OrderedMappings $OrderedMappings )) } return 'sequence:{0}:[{1}]' -f $items.Count, ($items -join '|') @@ -228,7 +229,10 @@ function ConvertTo-YamlSuiteReferenceSignature { [OutputType([string])] param ( [AllowNull()] - [object] $Value + [object] $Value, + + [AllowNull()] + [System.Collections.Generic.HashSet[object]] $OrderedMappings ) $idGenerator = [System.Runtime.Serialization.ObjectIDGenerator]::new() @@ -265,7 +269,8 @@ function ConvertTo-YamlSuiteReferenceSignature { if ($current -is [System.Collections.IDictionary]) { foreach ($entry in $current.GetEnumerator()) { $childPath = '{0}{{{1}}}' -f $frame.Path, ( - ConvertTo-YamlSuiteCanonicalValue -Value $entry.Key + ConvertTo-YamlSuiteCanonicalValue -Value $entry.Key ` + -OrderedMappings $OrderedMappings ) $stack.Push([pscustomobject]@{ Value = $entry.Value @@ -303,6 +308,49 @@ function ConvertTo-YamlSuiteReferenceSignature { return ($output -join ';') } +function Get-YamlSuiteDictionaryReferenceSet { + [OutputType([System.Collections.Generic.HashSet[object]])] + param ( + [AllowNull()] + [object] $Value + ) + + $comparer = [System.Collections.Generic.ReferenceEqualityComparer]::Instance + $dictionaries = [System.Collections.Generic.HashSet[object]]::new($comparer) + $visited = [System.Collections.Generic.HashSet[object]]::new($comparer) + $pending = [System.Collections.Generic.Stack[object]]::new() + $pending.Push($Value) + while ($pending.Count -gt 0) { + $current = $pending.Pop() + if ($null -eq $current -or $current -is [string] -or + $current.GetType().IsValueType) { + continue + } + if ($current -isnot [System.Collections.IDictionary] -and + $current -isnot [System.Collections.IEnumerable]) { + continue + } + if (-not $visited.Add($current)) { + continue + } + if ($current -is [System.Collections.IDictionary]) { + [void] $dictionaries.Add($current) + foreach ($entry in $current.GetEnumerator()) { + $pending.Push($entry.Key) + $pending.Push($entry.Value) + } + continue + } + if ($current -is [byte[]]) { + continue + } + foreach ($item in $current) { + $pending.Push($item) + } + } + Write-Output -InputObject $dictionaries -NoEnumerate +} + function Test-YamlSuiteBinaryByteArrayProjection { [OutputType([bool])] param ( @@ -738,24 +786,43 @@ $readYamlSuiteStream = { $projectYamlSuiteStream = { param ([object[]] $Nodes) $values = [System.Collections.Generic.List[object]]::new() + $orderedMappings = [System.Collections.Generic.HashSet[object]]::new( + [System.Collections.Generic.ReferenceEqualityComparer]::Instance + ) foreach ($node in $Nodes) { $cache = [System.Collections.Generic.Dictionary[int, object]]::new() $values.Add((ConvertFrom-YamlNode -Node $node -Cache $cache -AsHashtable).Value) + $visited = [System.Collections.Generic.HashSet[int]]::new() + $pending = [System.Collections.Generic.Stack[object]]::new() + $pending.Push($node) + while ($pending.Count -gt 0) { + $current = $pending.Pop() + while ($current.Kind -eq 'Alias') { + $current = $current.Target + } + if (-not $visited.Add($current.Id)) { + continue + } + if ($current.Tag -ceq 'tag:yaml.org,2002:omap' -and + $cache.ContainsKey($current.Id)) { + [void] $orderedMappings.Add($cache[$current.Id]) + } + if ($current.Kind -eq 'Sequence') { + foreach ($item in $current.Items) { + $pending.Push($item) + } + } elseif ($current.Kind -eq 'Mapping') { + foreach ($entry in $current.Entries) { + $pending.Push($entry.Key) + $pending.Push($entry.Value) + } + } + } } - New-YamlValueBox -Value ([object[]] $values.ToArray()) -} -$projectYamlSuiteText = { - param ([string] $YamlText) - - $stream = Read-YamlStream -Yaml $YamlText -Depth 128 -MaxNodes 100000 ` - -MaxAliases 1000 -MaxScalarLength 1048576 -MaxTagLength 1024 ` - -MaxTotalTagLength 65536 -MaxNumericLength 4096 - $values = [System.Collections.Generic.List[object]]::new() - foreach ($node in $stream.Value) { - $cache = [System.Collections.Generic.Dictionary[int, object]]::new() - $values.Add((ConvertFrom-YamlNode -Node $node -Cache $cache -AsHashtable).Value) - } - New-YamlValueBox -Value ([object[]] $values.ToArray()) + New-YamlValueBox -Value ([pscustomobject]@{ + Values = [object[]] $values.ToArray() + OrderedMappings = $orderedMappings + }) } $testYamlSuiteText = { param ([string] $YamlText) @@ -808,6 +875,7 @@ foreach ($inputFile in $inputFiles) { $representation = $null $stream = $null $projectedValues = $null + $projectedOrderedMappings = $null $projectedCanonical = $null $projectedJsonCanonical = $null $projectedReference = '' @@ -867,11 +935,20 @@ foreach ($inputFile in $inputFiles) { if ($null -ne $stream) { try { - $projectedValues = (Invoke-InYamlModule -ScriptBlock $projectYamlSuiteStream -Arguments (, $stream.Value)).Value - $projectedCanonical = ConvertTo-YamlSuiteCanonicalValue -Value ([object[]] $projectedValues) + $projected = ( + Invoke-InYamlModule -ScriptBlock $projectYamlSuiteStream -Arguments (, $stream.Value) + ).Value + $projectedValues = $projected.Values + $projectedOrderedMappings = $projected.OrderedMappings + $projectedCanonical = ConvertTo-YamlSuiteCanonicalValue ` + -Value ([object[]] $projectedValues) ` + -OrderedMappings $projectedOrderedMappings $projectedJsonCanonical = ConvertTo-YamlSuiteCanonicalValue ` - -Value ([object[]] $projectedValues) -SortMappings - $projectedReference = ConvertTo-YamlSuiteReferenceSignature -Value ([object[]] $projectedValues) + -Value ([object[]] $projectedValues) -SortMappings ` + -OrderedMappings $projectedOrderedMappings + $projectedReference = ConvertTo-YamlSuiteReferenceSignature ` + -Value ([object[]] $projectedValues) ` + -OrderedMappings $projectedOrderedMappings } catch { if ($_.Exception.Data.Contains('YamlErrorId')) { $projectionError = [string] $_.Exception.Data['YamlErrorId'] @@ -952,9 +1029,17 @@ foreach ($inputFile in $inputFiles) { $outYaml = [System.IO.File]::ReadAllText($outYamlPath, [System.Text.UTF8Encoding]::new($false, $true)) try { $outStream = Invoke-InYamlModule -ScriptBlock $readYamlSuiteStream -Arguments @($outYaml) - $outValues = (Invoke-InYamlModule -ScriptBlock $projectYamlSuiteStream -Arguments (, $outStream.Value)).Value - $outCanonical = ConvertTo-YamlSuiteCanonicalValue -Value ([object[]] $outValues) - $outReference = ConvertTo-YamlSuiteReferenceSignature -Value ([object[]] $outValues) + $outProjection = ( + Invoke-InYamlModule -ScriptBlock $projectYamlSuiteStream ` + -Arguments (, $outStream.Value) + ).Value + $outValues = $outProjection.Values + $outCanonical = ConvertTo-YamlSuiteCanonicalValue ` + -Value ([object[]] $outValues) ` + -OrderedMappings $outProjection.OrderedMappings + $outReference = ConvertTo-YamlSuiteReferenceSignature ` + -Value ([object[]] $outValues) ` + -OrderedMappings $outProjection.OrderedMappings $outYamlCanonical = $outCanonical $outYamlReference = $outReference if ($outCanonical -cne $projectedCanonical) { @@ -989,12 +1074,19 @@ foreach ($inputFile in $inputFiles) { $emitYamlResult = 'Fail' $emitYamlReason = 'EmitYamlInvalid' } else { - $fixtureValues = (Invoke-InYamlModule -ScriptBlock $projectYamlSuiteText ` - -Arguments @($emitYaml)).Value + $fixtureStream = Invoke-InYamlModule -ScriptBlock $readYamlSuiteStream ` + -Arguments @($emitYaml) + $fixtureProjection = ( + Invoke-InYamlModule -ScriptBlock $projectYamlSuiteStream ` + -Arguments (, $fixtureStream.Value) + ).Value + $fixtureValues = $fixtureProjection.Values $fixtureCanonical = ConvertTo-YamlSuiteCanonicalValue ` - -Value ([object[]] $fixtureValues) + -Value ([object[]] $fixtureValues) ` + -OrderedMappings $fixtureProjection.OrderedMappings $fixtureReference = ConvertTo-YamlSuiteReferenceSignature ` - -Value ([object[]] $fixtureValues) + -Value ([object[]] $fixtureValues) ` + -OrderedMappings $fixtureProjection.OrderedMappings $fixtureOracleCanonical = $null $fixtureOracleActual = $fixtureCanonical @@ -1005,9 +1097,16 @@ foreach ($inputFile in $inputFiles) { } elseif ($null -ne $jsonOracleCanonical) { $fixtureOracleCanonical = $jsonOracleCanonical $fixtureOracleActual = ConvertTo-YamlSuiteCanonicalValue ` - -Value ([object[]] $fixtureValues) -SortMappings + -Value ([object[]] $fixtureValues) -SortMappings ` + -OrderedMappings $fixtureProjection.OrderedMappings } + $emitYamlExpected = $fixtureOracleCanonical + $emitYamlActual = $fixtureOracleActual + if ($null -ne $projectedCanonical) { + $emitYamlExpectedReference = $projectedReference + } + $emitYamlActualReference = $fixtureReference if ($null -eq $fixtureOracleCanonical) { $emitYamlResult = 'Fail' $emitYamlReason = 'EmitYamlOracleUnavailable' @@ -1015,54 +1114,10 @@ foreach ($inputFile in $inputFiles) { $fixtureReferenceMismatch) { $emitYamlResult = 'Fail' $emitYamlReason = 'EmitYamlRepresentationMismatch' - $emitYamlExpected = $fixtureOracleCanonical - $emitYamlActual = $fixtureOracleActual - if ($null -ne $projectedCanonical) { - $emitYamlExpectedReference = $projectedReference - } - $emitYamlActualReference = $fixtureReference } else { - $fixtureEmittedDocuments = [System.Collections.Generic.List[string]]::new() - foreach ($fixtureValue in $fixtureValues) { - $fixtureEmitted = Invoke-InYamlModule -ScriptBlock { - param ($InputValue) - ConvertTo-Yaml -InputObject $InputValue -ExplicitDocumentStart - } -Arguments (, $fixtureValue) - $fixtureEmittedDocuments.Add([string] $fixtureEmitted) - } - $fixtureEmittedText = $fixtureEmittedDocuments.ToArray() -join "`n" - $isValidFixtureEmit = Invoke-InYamlModule ` - -ScriptBlock $testYamlSuiteText -Arguments @($fixtureEmittedText) - if (-not $isValidFixtureEmit) { - $emitYamlResult = 'Fail' - $emitYamlReason = 'EmittedYamlInvalid' - } else { - $fixtureRoundTripValues = ( - Invoke-InYamlModule -ScriptBlock $projectYamlSuiteText ` - -Arguments @($fixtureEmittedText) - ).Value - $fixtureRoundCanonical = ConvertTo-YamlSuiteCanonicalValue ` - -Value ([object[]] $fixtureRoundTripValues) - $fixtureRoundReference = ConvertTo-YamlSuiteReferenceSignature ` - -Value ([object[]] $fixtureRoundTripValues) - $emitYamlExpected = $fixtureCanonical - $emitYamlActual = $fixtureRoundCanonical - $emitYamlExpectedReference = $fixtureReference - $emitYamlActualReference = $fixtureRoundReference - } - if ($emitYamlResult -ne 'Fail' -and - $fixtureRoundCanonical -ceq $fixtureCanonical -and - $fixtureRoundReference -ceq $fixtureReference) { - $emitYamlResult = 'Pass' - } elseif ($emitYamlResult -ne 'Fail') { - $emitYamlResult = 'Fail' - $emitYamlReason = 'EmitYamlRoundTripMismatch' - } + $emitYamlResult = 'Pass' } } - } catch [System.NotSupportedException] { - $emitYamlResult = 'Fail' - $emitYamlReason = 'UnsupportedEmissionType' } catch { if ($_.Exception.Data.Contains('IsYamlException')) { $emitYamlResult = 'Fail' @@ -1100,17 +1155,50 @@ foreach ($inputFile in $inputFiles) { $selfRoundTripResult = 'Fail' $selfRoundTripReason = 'EmittedYamlInvalid' } else { - $roundTripValues = (Invoke-InYamlModule -ScriptBlock $projectYamlSuiteText ` - -Arguments @($emittedText)).Value - $roundCanonical = ConvertTo-YamlSuiteCanonicalValue -Value ([object[]] $roundTripValues) - $roundReference = ConvertTo-YamlSuiteReferenceSignature -Value ([object[]] $roundTripValues) + $roundTripStream = Invoke-InYamlModule -ScriptBlock $readYamlSuiteStream ` + -Arguments @($emittedText) + $roundTripProjection = ( + Invoke-InYamlModule -ScriptBlock $projectYamlSuiteStream ` + -Arguments (, $roundTripStream.Value) + ).Value + $roundTripValues = $roundTripProjection.Values + $roundCanonical = ConvertTo-YamlSuiteCanonicalValue ` + -Value ([object[]] $roundTripValues) ` + -OrderedMappings $roundTripProjection.OrderedMappings + $roundReference = ConvertTo-YamlSuiteReferenceSignature ` + -Value ([object[]] $roundTripValues) ` + -OrderedMappings $roundTripProjection.OrderedMappings $selfRoundTripCanonical = $roundCanonical $selfRoundTripReference = $roundReference if ($roundCanonical -ceq $projectedCanonical -and $roundReference -ceq $projectedReference) { $selfRoundTripResult = 'Pass' } else { - $selfRoundTripResult = 'Fail' - $selfRoundTripReason = 'SelfRoundTripMismatch' + $projectedDictionaryMappings = Get-YamlSuiteDictionaryReferenceSet ` + -Value ([object[]] $projectedValues) + $roundDictionaryMappings = Get-YamlSuiteDictionaryReferenceSet ` + -Value ([object[]] $roundTripValues) + $projectedProjectionCanonical = ConvertTo-YamlSuiteCanonicalValue ` + -Value ([object[]] $projectedValues) ` + -OrderedMappings $projectedDictionaryMappings + $roundProjectionCanonical = ConvertTo-YamlSuiteCanonicalValue ` + -Value ([object[]] $roundTripValues) ` + -OrderedMappings $roundDictionaryMappings + $projectedProjectionReference = ConvertTo-YamlSuiteReferenceSignature ` + -Value ([object[]] $projectedValues) ` + -OrderedMappings $projectedDictionaryMappings + $roundProjectionReference = ConvertTo-YamlSuiteReferenceSignature ` + -Value ([object[]] $roundTripValues) ` + -OrderedMappings $roundDictionaryMappings + if ($projectedOrderedMappings.Count -gt + $roundTripProjection.OrderedMappings.Count -and + $projectedProjectionCanonical -ceq $roundProjectionCanonical -and + $projectedProjectionReference -ceq $roundProjectionReference) { + $selfRoundTripResult = 'PolicyDifference' + $selfRoundTripReason = 'LegacyOrderedMapProjection' + } else { + $selfRoundTripResult = 'Fail' + $selfRoundTripReason = 'SelfRoundTripMismatch' + } } } } catch [System.NotSupportedException] { From 63961d7cde1e58c24c86deb4d9b1d34d5bf05178 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 24 Jul 2026 02:34:53 +0200 Subject: [PATCH 21/24] Document ordered-map conformance policy Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e0d8502..9b058eb 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ The archive contains 402 inputs: | JSON projection | 277 | 2 | 0 | 123 | | `out.yaml` projection | 241 | 1 | 0 | 160 | | Official `emit.yaml` fixtures | 55 | 0 | 0 | 347 | -| Module self-round-trip | 306 | 2 | 0 | 94 | +| Module self-round-trip | 305 | 3 | 0 | 94 | All 94 fixtures marked invalid are rejected. The valid `2JQS` and `X38W` inputs are syntactically recognized and produce matching representation @@ -189,10 +189,15 @@ case with an `out.yaml` fixture, is also reported that way on that surface. The two JSON projection differences are `565N`, where `!!binary` intentionally becomes `byte[]` instead of a Base64 string, and `J7PZ`, where legacy `!!omap` intentionally becomes `System.Collections.Specialized.OrderedDictionary` -instead of remaining a sequence of one-entry mappings. No event mismatch is -classified as policy. All 55 official `emit.yaml` fixtures are read and -validated independently of the 402-input module self-round-trip. The -deterministic runner reports no unexplained failures. +instead of remaining a sequence of one-entry mappings. `J7PZ` is also a +self-round-trip policy difference: once projected, its ordered dictionary +cannot be distinguished from an ordinary insertion-ordered PowerShell +dictionary, so emission cannot reconstruct the explicit `!!omap` tag. The +runner still preserves and compares `!!omap` order from representation +metadata. No event mismatch is classified as policy. All 55 official +`emit.yaml` fixtures are read and validated independently of the 402-input +module self-round-trip. The deterministic runner reports no unexplained +failures. These results are a pinned compatibility measurement, not a claim that a finite corpus proves complete YAML 1.2.2 compliance. From 28ac74e1cd96c4112cb41ae27814c5df77143173 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 24 Jul 2026 02:51:56 +0200 Subject: [PATCH 22/24] Avoid quadratic BOM prefix scans Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Read-YamlDocumentByteOrderMark.ps1 | 18 +++++++++----- .../private/Skip-YamlDocumentPrefix.ps1 | 24 +++++++++++++++---- tests/ConvertFrom-Yaml.Tests.ps1 | 19 +++++++++++++++ 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/src/functions/private/Read-YamlDocumentByteOrderMark.ps1 b/src/functions/private/Read-YamlDocumentByteOrderMark.ps1 index 64334d7..e44bc2b 100644 --- a/src/functions/private/Read-YamlDocumentByteOrderMark.ps1 +++ b/src/functions/private/Read-YamlDocumentByteOrderMark.ps1 @@ -12,7 +12,9 @@ function Read-YamlDocumentByteOrderMark { [Parameter(Mandatory)] [pscustomobject] $Context, - [switch] $RequireDocumentStart + [switch] $RequireDocumentStart, + + [switch] $SkipValidation ) if ($Context.LineIndex -ge $Context.Lines.Count) { @@ -20,7 +22,7 @@ function Read-YamlDocumentByteOrderMark { } $index = $Context.LineStarts[$Context.LineIndex] $atStreamStart = $index -eq 0 - if (-not $atStreamStart -and + if (-not $atStreamStart -and -not $SkipValidation -and -not (Test-YamlDocumentByteOrderMark -Context $Context ` -RequireDocumentStart:$RequireDocumentStart)) { return @@ -32,9 +34,13 @@ function Read-YamlDocumentByteOrderMark { return } - $Context.Text = $Context.Text.Remove($index, 1) - $Context.Lines[$Context.LineIndex] = $Context.Lines[$Context.LineIndex].Substring(1) - for ($line = $Context.LineIndex + 1; $line -lt $Context.LineStarts.Count; $line++) { - $Context.LineStarts[$line]-- + $line = $Context.Lines[$Context.LineIndex] + $count = 0 + while ($count -lt $line.Length -and $line[$count] -ceq [char] 0xFEFF) { + $count++ } + + # Keep the source immutable and advance only this line's physical source offset. + $Context.Lines[$Context.LineIndex] = $line.Substring($count) + $Context.LineStarts[$Context.LineIndex] += $count } diff --git a/src/functions/private/Skip-YamlDocumentPrefix.ps1 b/src/functions/private/Skip-YamlDocumentPrefix.ps1 index 481a881..39ad48d 100644 --- a/src/functions/private/Skip-YamlDocumentPrefix.ps1 +++ b/src/functions/private/Skip-YamlDocumentPrefix.ps1 @@ -15,14 +15,30 @@ function Skip-YamlDocumentPrefix { [switch] $RequireDocumentStart ) + $validatedPrefix = $false do { $lineIndex = $Context.LineIndex - $textLength = $Context.Text.Length - Read-YamlDocumentByteOrderMark -Context $Context ` - -RequireDocumentStart:$RequireDocumentStart + $consumedByteOrderMark = $false + if ($lineIndex -lt $Context.Lines.Count -and + $Context.Lines[$lineIndex].StartsWith( + [string] [char] 0xFEFF, + [System.StringComparison]::Ordinal + )) { + $atStreamStart = $Context.LineStarts[$lineIndex] -eq 0 + if ($atStreamStart -or $validatedPrefix -or + (Test-YamlDocumentByteOrderMark -Context $Context ` + -RequireDocumentStart:$RequireDocumentStart)) { + Read-YamlDocumentByteOrderMark -Context $Context ` + -RequireDocumentStart:$RequireDocumentStart -SkipValidation + $consumedByteOrderMark = $true + if (-not $atStreamStart) { + $validatedPrefix = $true + } + } + } Skip-YamlBlockTrivia -Context $Context } while ( $Context.LineIndex -ne $lineIndex -or - $Context.Text.Length -ne $textLength + $consumedByteOrderMark ) } diff --git a/tests/ConvertFrom-Yaml.Tests.ps1 b/tests/ConvertFrom-Yaml.Tests.ps1 index 93add1f..c904232 100644 --- a/tests/ConvertFrom-Yaml.Tests.ps1 +++ b/tests/ConvertFrom-Yaml.Tests.ps1 @@ -249,6 +249,25 @@ folded: > ("[value # a${bom}b`n ]" | Test-Yaml) | Should -BeFalse ("| # a${bom}b`n value" | Test-Yaml) | Should -BeFalse } + + It 'validates repeated document prefixes with one suffix scan' { + $bom = [char] 0xFEFF + $yaml = ((([string] $bom + "# prefix`n") * 64) -join '') + "---`nvalue" + $loadedModule = Get-Module -Name Yaml | Select-Object -First 1 + if ($null -eq $loadedModule) { + Mock Test-YamlDocumentPrefix { $true } + } else { + Mock Test-YamlDocumentPrefix -ModuleName $loadedModule.Name { $true } + } + + ($yaml | ConvertFrom-Yaml) | Should -Be 'value' + if ($null -eq $loadedModule) { + Should -Invoke Test-YamlDocumentPrefix -Times 1 -Exactly + } else { + Should -Invoke Test-YamlDocumentPrefix -ModuleName $loadedModule.Name ` + -Times 1 -Exactly + } + } } Context 'Tags, anchors, and aliases' { From 995a6b90f8ae8c8d28f034d32ccd2f9eed3d654d Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 24 Jul 2026 03:20:41 +0200 Subject: [PATCH 23/24] Allow BOM prefixes before bare documents Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/functions/private/Test-YamlDocumentPrefix.ps1 | 8 ++++---- tests/ConvertFrom-Yaml.Tests.ps1 | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/functions/private/Test-YamlDocumentPrefix.ps1 b/src/functions/private/Test-YamlDocumentPrefix.ps1 index 9354b2a..2b81a2f 100644 --- a/src/functions/private/Test-YamlDocumentPrefix.ps1 +++ b/src/functions/private/Test-YamlDocumentPrefix.ps1 @@ -45,11 +45,11 @@ function Test-YamlDocumentPrefix { $Index = $lineEnd + 1 continue } - if (-not $line.StartsWith('---', [System.StringComparison]::Ordinal)) { - return $false + if ($line.StartsWith('---', [System.StringComparison]::Ordinal) -and + ($line.Length -eq 3 -or (Test-YamlWhiteSpace -Character $line[3]))) { + return $true } - return $line.Length -eq 3 -or - (Test-YamlWhiteSpace -Character $line[3]) + return -not $RequireDocumentStart -and -not $directiveSeen } return -not $directiveSeen } diff --git a/tests/ConvertFrom-Yaml.Tests.ps1 b/tests/ConvertFrom-Yaml.Tests.ps1 index c904232..659bf4d 100644 --- a/tests/ConvertFrom-Yaml.Tests.ps1 +++ b/tests/ConvertFrom-Yaml.Tests.ps1 @@ -228,16 +228,29 @@ folded: > $trailingPrefix = @( "---`nvalue`n...`n${bom}# trailing prefix" | ConvertFrom-Yaml ) + $barePrefixDocument = @( + "# prefix`n${bom}value" | ConvertFrom-Yaml + ) + $bareBoundaryDocuments = @( + "---`none`n...`n${bom}two" | ConvertFrom-Yaml + ) + $commentedBareBoundaryDocuments = @( + "---`none`n...`n${bom}# comment`ntwo" | ConvertFrom-Yaml + ) $documents | Should -Be @('one', 'two') $implicitBoundaryDocuments | Should -Be @('one', 'two') $commentThenBom | Should -Be @('value') $alternatingPrefixes | Should -Be @('value') $trailingPrefix | Should -Be @('value') + $barePrefixDocument | Should -Be @('value') + $bareBoundaryDocuments | Should -Be @('one', 'two') + $commentedBareBoundaryDocuments | Should -Be @('one', 'two') ("${bom}---`nvalue" | ConvertFrom-Yaml) | Should -Be 'value' ("foo${bom}bar" | Test-Yaml) | Should -BeFalse ("---`n${bom}value" | Test-Yaml) | Should -BeFalse - ("---`none`n...`n${bom}# comment`ntwo" | Test-Yaml) | Should -BeFalse + ("---`none`n${bom}two" | Test-Yaml) | Should -BeFalse + ("${bom}%YAML 1.2`nvalue" | Test-Yaml) | Should -BeFalse ('"foo' + $bom + 'bar"' | ConvertFrom-Yaml) | Should -Be "foo${bom}bar" ("'foo${bom}bar'" | ConvertFrom-Yaml) | Should -Be "foo${bom}bar" ("`"a`n${bom}%foo`nb`"" | ConvertFrom-Yaml) | From 0c2fcc16e942ac282bb5fb2ecba4441243c16ea4 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 24 Jul 2026 03:20:58 +0200 Subject: [PATCH 24/24] Distinguish ordered maps in conformance Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/Conformance.Tests.ps1 | 33 ++++++++++++++++++++++++++++ tests/tools/Invoke-YamlTestSuite.ps1 | 3 ++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/Conformance.Tests.ps1 b/tests/Conformance.Tests.ps1 index 16810ec..bda0a57 100644 --- a/tests/Conformance.Tests.ps1 +++ b/tests/Conformance.Tests.ps1 @@ -167,6 +167,9 @@ Describe 'Released yaml-test-suite corpus accounting' { $firstOrder.Add('b', 2) $secondOrder.Add('b', 2) $secondOrder.Add('a', 1) + $ordinarySameOrder = [System.Collections.Specialized.OrderedDictionary]::new() + $ordinarySameOrder.Add('a', 1) + $ordinarySameOrder.Add('b', 2) (ConvertTo-YamlSuiteCanonicalValue -Value $firstOrder) | Should -Be (ConvertTo-YamlSuiteCanonicalValue -Value $secondOrder) @@ -181,6 +184,12 @@ Describe 'Released yaml-test-suite corpus accounting' { ConvertTo-YamlSuiteCanonicalValue -Value $secondOrder ` -OrderedMappings $orderedMappings ) + (ConvertTo-YamlSuiteCanonicalValue -Value $firstOrder ` + -OrderedMappings $orderedMappings) | + Should -Not -Be ( + ConvertTo-YamlSuiteCanonicalValue -Value $ordinarySameOrder ` + -OrderedMappings $orderedMappings + ) $firstNestedKey = [System.Collections.Specialized.OrderedDictionary]::new() $secondNestedKey = [System.Collections.Specialized.OrderedDictionary]::new() @@ -272,6 +281,30 @@ ship-to: $ordinaryResult.OutYamlResult | Should -Be 'Pass' } + It 'detects explicit ordered-map tag loss across representation surfaces' { + $tagLossSuitePath = Join-Path $TestDrive 'ordered-map-tag-loss' + $tagLossCasePath = Join-Path $tagLossSuitePath 'omap-tag-loss' + $null = New-Item -Path $tagLossCasePath -ItemType Directory -Force + "!!omap`n- a: 1`n- b: 2" | + Set-Content -LiteralPath (Join-Path $tagLossCasePath 'in.yaml') ` + -Encoding utf8NoBOM + foreach ($fixture in @('out.yaml', 'emit.yaml')) { + "a: 1`nb: 2" | + Set-Content -LiteralPath (Join-Path $tagLossCasePath $fixture) ` + -Encoding utf8NoBOM + } + + $tagLossResult = & $runnerPath -Path $tagLossSuitePath ` + -CompareOutYaml -CompareEmitYaml -CompareSelfRoundTrip + + $tagLossResult.OutYamlResult | Should -Be 'Fail' + $tagLossResult.OutYamlReason | Should -Be 'OutYamlConstructionMismatch' + $tagLossResult.EmitYamlResult | Should -Be 'Fail' + $tagLossResult.EmitYamlReason | Should -Be 'EmitYamlRepresentationMismatch' + $tagLossResult.SelfRoundTripResult | Should -Be 'PolicyDifference' + $tagLossResult.SelfRoundTripReason | Should -Be 'LegacyOrderedMapProjection' + } + It 'accounts for out.yaml representation comparisons' { @($suiteResults | Where-Object OutYamlResult -EQ 'Pass').Count | Should -Be 241 @($suiteResults | Where-Object OutYamlResult -EQ 'PolicyDifference').Count | diff --git a/tests/tools/Invoke-YamlTestSuite.ps1 b/tests/tools/Invoke-YamlTestSuite.ps1 index a0be294..77c1b63 100644 --- a/tests/tools/Invoke-YamlTestSuite.ps1 +++ b/tests/tools/Invoke-YamlTestSuite.ps1 @@ -199,7 +199,8 @@ function ConvertTo-YamlSuiteCanonicalValue { if ($SortMappings -or -not $isOrdered) { $entries.Sort([System.StringComparer]::Ordinal) } - return 'map:{0}:{{{1}}}' -f $entries.Count, ($entries -join '|') + $mappingKind = if ($isOrdered) { 'omap' } else { 'map' } + return '{0}:{1}:{{{2}}}' -f $mappingKind, $entries.Count, ($entries -join '|') } if ($Value -is [System.Collections.IEnumerable] -and $Value -isnot [string]) { $items = [System.Collections.Generic.List[string]]::new()