Harden YAML engine and conformance accounting

README.md

@@ -175,19 +175,28 @@ The archive contains 402 inputs:
 | Syntax and composition | 400 | 2 | 0 | 0 |
 | Representation events | 308 | 0 | 0 | 94 |
 | JSON projection | 277 | 2 | 0 | 123 |
-| `out.yaml` projection | 241 | 0 | 0 | 161 |
-| Emit and round trip | 306 | 0 | 0 | 96 |
+| `out.yaml` projection | 241 | 1 | 0 | 160 |
+| Official `emit.yaml` fixtures | 55 | 0 | 0 | 347 |
+| Module self-round-trip | 305 | 3 | 0 | 94 |
 
 All 94 fixtures marked invalid are rejected. The valid `2JQS` and `X38W`
 inputs are syntactically recognized and produce matching representation
 events, then are rejected during load validation because representation
-mapping keys must be unique. They are not unsupported grammar.
+mapping keys must be unique. They are not unsupported grammar. Both are
+reported as policy differences for module self-round-trip; `X38W`, the one
+case with an `out.yaml` fixture, is also reported that way on that surface.
 
 The two JSON projection differences are `565N`, where `!!binary` intentionally
 becomes `byte[]` instead of a Base64 string, and `J7PZ`, where legacy `!!omap`
 intentionally becomes `System.Collections.Specialized.OrderedDictionary`
-instead of remaining a sequence of one-entry mappings. No event mismatch is
-classified as policy. The deterministic runner reports no unexplained
+instead of remaining a sequence of one-entry mappings. `J7PZ` is also a
+self-round-trip policy difference: once projected, its ordered dictionary
+cannot be distinguished from an ordinary insertion-ordered PowerShell
+dictionary, so emission cannot reconstruct the explicit `!!omap` tag. The
+runner still preserves and compares `!!omap` order from representation
+metadata. No event mismatch is classified as policy. All 55 official
+`emit.yaml` fixtures are read and validated independently of the 402-input
+module self-round-trip. The deterministic runner reports no unexplained
 failures.
 
 These results are a pinned compatibility measurement, not a claim that a finite

src/functions/private/Add-YamlDictionaryEntry.ps1

@@ -0,0 +1,37 @@
+function Add-YamlDictionaryEntry {
+    <#
+        .SYNOPSIS
+        Adds a projected mapping entry with a YAML-classified collision error.
+    #>
+    [Diagnostics.CodeAnalysis.SuppressMessageAttribute(
+        'PSUseShouldProcessForStateChangingFunctions', '',
+        Justification = 'Mutates an internal projection dictionary.'
+    )]
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory)]
+        [System.Collections.IDictionary] $Dictionary,
+
+        [Parameter(Mandatory)]
+        [object] $Key,
+
+        [Parameter(Mandatory)]
+        [AllowNull()]
+        [object] $Value,
+
+        [Parameter(Mandatory)]
+        [pscustomobject] $KeyNode
+    )
+
+    try {
+        $Dictionary.Add($Key, $Value)
+    } catch [System.Management.Automation.MethodInvocationException] {
+        if ($_.Exception.InnerException -isnot [System.ArgumentException]) {
+            throw
+        }
+        throw (New-YamlException -Start $KeyNode.Start -End $KeyNode.End `
+                -ErrorId 'YamlProjectionKeyCollision' -Message (
+                'Distinct YAML mapping keys cannot be projected distinctly into a PowerShell dictionary.'
+            ))
+    }
+}

src/functions/private/Assert-YamlNoByteOrderMark.ps1

@@ -0,0 +1,21 @@
+function Assert-YamlNoByteOrderMark {
+    <#
+        .SYNOPSIS
+        Rejects a raw byte order mark outside a quoted scalar or document prefix.
+    #>
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory)]
+        [AllowEmptyString()]
+        [string] $Text,
+
+        [Parameter(Mandatory)]
+        [pscustomobject] $Mark
+    )
+
+    if ($Text.IndexOf([char] 0xFEFF) -ge 0) {
+        throw (New-YamlException -Start $Mark -End $Mark -ErrorId 'YamlInvalidByteOrderMark' -Message (
+                'A raw YAML byte order mark is only allowed in a quoted scalar or document prefix.'
+            ))
+    }
+}

src/functions/private/ConvertFrom-YamlNode.ps1

@@ -154,7 +154,9 @@ function ConvertFrom-YamlNode {
             continue
         }
         if ($frame.State -eq 'OmapValue') {
-            $frame.Result.Add($frame.Key, $frame.Child.PSObject.Properties['Value'].Value)
+            Add-YamlDictionaryEntry -Dictionary $frame.Result -Key $frame.Key `
+                -Value $frame.Child.PSObject.Properties['Value'].Value `
+                -KeyNode $frame.Node.Items[$frame.Index].Entries[0].Key
             $frame.Index++
             $frame.State = 'OmapKey'
             continue
@@ -188,7 +190,8 @@ function ConvertFrom-YamlNode {
                 $frame.Key = $keyValue
             }
             if ($frame.Node.Tag -ceq 'tag:yaml.org,2002:set') {
-                $frame.Result.Add($frame.Key, $null)
+                Add-YamlDictionaryEntry -Dictionary $frame.Result -Key $frame.Key -Value $null `
+                    -KeyNode $frame.Node.Entries[$frame.Index].Key
                 $frame.Index++
                 $frame.State = 'DictionaryKey'
                 continue
@@ -209,7 +212,9 @@ function ConvertFrom-YamlNode {
             continue
         }
         if ($frame.State -eq 'DictionaryValue') {
-            $frame.Result.Add($frame.Key, $frame.Child.PSObject.Properties['Value'].Value)
+            Add-YamlDictionaryEntry -Dictionary $frame.Result -Key $frame.Key `
+                -Value $frame.Child.PSObject.Properties['Value'].Value `
+                -KeyNode $frame.Node.Entries[$frame.Index].Key
             $frame.Index++
             $frame.State = 'DictionaryKey'
             continue
@@ -244,6 +249,13 @@ function ConvertFrom-YamlNode {
                         'This mapping key cannot be represented as a PSCustomObject property. Use -AsHashtable.'
                     ))
             }
+            if (Test-YamlReservedPropertyName -Name $frame.Key) {
+                $keyNode = $frame.Node.Entries[$frame.Index].Key
+                throw (New-YamlException -Start $keyNode.Start -End $keyNode.End `
+                        -ErrorId 'YamlPropertyNameReserved' -Message (
+                        "The mapping key '$($frame.Key)' is reserved by PowerShell ETS. Use -AsHashtable."
+                    ))
+            }
             if (-not $frame.Names.Add($frame.Key)) {
                 $keyNode = $frame.Node.Entries[$frame.Index].Key
                 throw (New-YamlException -Start $keyNode.Start -End $keyNode.End `

src/functions/private/ConvertTo-YamlFlowText.ps1

@@ -132,7 +132,11 @@ function ConvertTo-YamlFlowText {
             continue
         }
         if ($frame.State -eq 'MappingValue') {
-            $frame.Parts.Add("$($frame.KeyText)`: $($frame.Child.Value)")
+            $explicitKey = (
+                Get-YamlEmissionImplicitKeyLength -RenderedText $frame.KeyText
+            ) -gt 1024
+            $keyPrefix = if ($explicitKey) { '? ' } else { '' }
+            $frame.Parts.Add("$keyPrefix$($frame.KeyText)`: $($frame.Child.Value)")
             $frame.Index++
             $frame.State = 'MappingKey'
         }

src/functions/private/ConvertTo-YamlNode.ps1

@@ -34,6 +34,7 @@ function ConvertTo-YamlNode {
             Child        = $null
             KeyNode      = $null
             Fingerprints = $null
+            Reserved     = $false
         })
 
     while ($stack.Count -gt 0) {
@@ -44,13 +45,35 @@ function ConvertTo-YamlNode {
                         "The object graph exceeds the configured depth limit of $($State.MaxDepth)."
                     ))
             }
+            if ($frame.Reserved) {
+                $State.ReservedNodeCount--
+            }
             $State.NodeCount++
             if ($State.NodeCount -gt $State.MaxNodes) {
                 throw (New-YamlSerializationException -ErrorId 'YamlNodeLimitExceeded' -Message (
                         "The object graph exceeds the configured limit of $($State.MaxNodes) nodes."
                     ))
             }
 
+            $isReference = Test-YamlSerializationReference -Value $frame.Value
+            if ($isReference) {
+                $firstTime = $false
+                $frame.ReferenceId = $State.IdGenerator.GetId($frame.Value, [ref] $firstTime)
+                if (-not $firstTime) {
+                    $State.ReferenceCounts[$frame.ReferenceId]++
+                    if ($State.Active.Contains($frame.ReferenceId)) {
+                        throw (New-YamlSerializationException -ErrorId 'YamlCycleDetected' -Message (
+                                "A cycle was detected while serializing type '$($frame.Value.GetType().FullName)'."
+                            ))
+                    }
+                    $frame.Holder.Value = $State.NodesById[$frame.ReferenceId]
+                    [void] $stack.Pop()
+                    continue
+                }
+                $State.ReferenceCounts[$frame.ReferenceId] = 1
+                $State.ReferenceOrder.Add($frame.ReferenceId)
+            }
+
             $frame.Shape = Get-YamlSerializationShape -Value $frame.Value -State $State `
                 -EnumsAsStrings:$EnumsAsStrings
             if ($frame.Shape.Kind -eq 'Scalar') {
@@ -59,22 +82,6 @@ function ConvertTo-YamlNode {
                 continue
             }
 
-            $firstTime = $false
-            $frame.ReferenceId = $State.IdGenerator.GetId($frame.Value, [ref] $firstTime)
-            if (-not $firstTime) {
-                $State.ReferenceCounts[$frame.ReferenceId]++
-                if ($State.Active.Contains($frame.ReferenceId)) {
-                    throw (New-YamlSerializationException -ErrorId 'YamlCycleDetected' -Message (
-                            "A cycle was detected while serializing type '$($frame.Value.GetType().FullName)'."
-                        ))
-                }
-                $frame.Holder.Value = $State.NodesById[$frame.ReferenceId]
-                [void] $stack.Pop()
-                continue
-            }
-
-            $State.ReferenceCounts[$frame.ReferenceId] = 1
-            $State.ReferenceOrder.Add($frame.ReferenceId)
             if ($frame.Shape.Kind -eq 'Binary') {
                 $frame.Shape.Node.ReferenceId = $frame.ReferenceId
                 $State.NodesById[$frame.ReferenceId] = $frame.Shape.Node
@@ -119,6 +126,7 @@ function ConvertTo-YamlNode {
                     Child        = $null
                     KeyNode      = $null
                     Fingerprints = $null
+                    Reserved     = $true
                 })
             continue
         }
@@ -149,6 +157,7 @@ function ConvertTo-YamlNode {
                     Child        = $null
                     KeyNode      = $null
                     Fingerprints = $null
+                    Reserved     = $true
                 })
             continue
         }
@@ -177,6 +186,7 @@ function ConvertTo-YamlNode {
                     Child        = $null
                     KeyNode      = $null
                     Fingerprints = $null
+                    Reserved     = $true
                 })
             continue
         }

src/functions/private/ConvertTo-YamlQuotedText.ps1

@@ -50,7 +50,7 @@ function ConvertTo-YamlQuotedText {
             [void] $builder.Append('\"')
         } elseif ($code -eq 92) {
             [void] $builder.Append('\\')
-        } elseif ($code -lt 0x20 -or $code -eq 0x7F -or
+        } elseif ($code -lt 0x20 -or $code -eq 0x7F -or $code -eq 0xFEFF -or
             $code -eq 0xFFFE -or $code -eq 0xFFFF -or
             ($code -ge 0x80 -and $code -le 0x9F)) {
             [void] $builder.Append(('\u{0:X4}' -f $code))

src/functions/private/ConvertTo-YamlTimestampText.ps1

@@ -0,0 +1,51 @@
+function ConvertTo-YamlTimestampText {
+    <#
+        .SYNOPSIS
+        Formats a representable CLR timestamp for YAML emission.
+    #>
+    [CmdletBinding()]
+    [OutputType([string])]
+    param (
+        [Parameter(Mandatory)]
+        [object] $Value
+    )
+
+    try {
+        if ($Value -is [datetimeoffset]) {
+            $offsetMinutes = $Value.Offset.TotalMinutes
+            if ($offsetMinutes -lt -840 -or $offsetMinutes -gt 840 -or
+                $offsetMinutes -ne [System.Math]::Truncate($offsetMinutes) -or
+                $Value.UtcTicks -lt [datetime]::MinValue.Ticks -or
+                $Value.UtcTicks -gt [datetime]::MaxValue.Ticks) {
+                throw [System.ArgumentOutOfRangeException]::new('Value')
+            }
+            return $Value.ToString(
+                'o',
+                [System.Globalization.CultureInfo]::InvariantCulture
+            )
+        }
+
+        if ($Value.Kind -eq [System.DateTimeKind]::Local) {
+            return [datetimeoffset]::new($Value).ToString(
+                'o',
+                [System.Globalization.CultureInfo]::InvariantCulture
+            )
+        }
+
+        $utc = if ($Value.Kind -eq [System.DateTimeKind]::Utc) {
+            $Value
+        } else {
+            [datetime]::SpecifyKind($Value, [System.DateTimeKind]::Utc)
+        }
+        return $utc.ToString(
+            "yyyy-MM-dd'T'HH:mm:ss.fffffff'Z'",
+            [System.Globalization.CultureInfo]::InvariantCulture
+        )
+    } catch [System.ArgumentException] {
+        throw (New-YamlSerializationException -ErrorId 'YamlTimestampSerializationFailed' `
+                -Message 'The timestamp cannot be represented with its local or explicit UTC offset.')
+    } catch [System.FormatException] {
+        throw (New-YamlSerializationException -ErrorId 'YamlTimestampSerializationFailed' `
+                -Message 'The timestamp cannot be represented with its local or explicit UTC offset.')
+    }
+}

src/functions/private/Find-YamlCommentStart.ps1

@@ -0,0 +1,22 @@
+function Find-YamlCommentStart {
+    <#
+        .SYNOPSIS
+        Finds a comment indicator separated by YAML s-white.
+    #>
+    [CmdletBinding()]
+    [OutputType([int])]
+    param (
+        [Parameter(Mandatory)]
+        [AllowEmptyString()]
+        [string] $Text
+    )
+
+    for ($index = 0; $index -lt $Text.Length; $index++) {
+        if ($Text[$index] -ceq '#' -and (
+                $index -eq 0 -or (Test-YamlWhiteSpace -Character $Text[$index - 1])
+            )) {
+            return $index
+        }
+    }
+    return -1
+}