Parse UTF-8 streams as bytes for precise error offsets

Author: xoofx

src/Tomlyn.Tests/NewApiUtf8StreamExceptionLocationTests.cs

@@ -0,0 +1,52 @@
+using System.IO;
+using NUnit.Framework;
+using Tomlyn.Model;
+
+namespace Tomlyn.Tests;
+
+public sealed class NewApiUtf8StreamExceptionLocationTests
+{
+    [Test]
+    public void Deserialize_Utf8Stream_InvalidUtf8_ThrowsTomlExceptionWithByteOffset()
+    {
+        // a = "<invalid utf-8>"
+        var bytes = new byte[]
+        {
+            (byte)'a',
+            (byte)' ',
+            (byte)'=',
+            (byte)' ',
+            (byte)'"',
+            0xC3, // invalid sequence start (expects continuation byte in 0x80..0xBF, but 0x28 is '(')
+            0x28,
+            (byte)'"',
+        };
+
+        using var stream = new MemoryStream(bytes);
+
+        var ex = Assert.Throws<TomlException>(() => TomlSerializer.Deserialize<TomlTable>(stream));
+        Assert.NotNull(ex);
+        Assert.NotNull(ex!.Span);
+
+        Assert.AreEqual(5, ex.Offset, "Offset must be the byte position of the invalid sequence.");
+        Assert.AreEqual(1, ex.Line);
+        Assert.AreEqual(6, ex.Column);
+    }
+
+    [Test]
+    public void Deserialize_Utf8Stream_WithBom_Succeeds()
+    {
+        var bytes = new byte[]
+        {
+            0xEF, 0xBB, 0xBF, // UTF-8 BOM
+            (byte)'a', (byte)' ', (byte)'=', (byte)' ', (byte)'1', (byte)'\n',
+        };
+
+        using var stream = new MemoryStream(bytes);
+        var table = TomlSerializer.Deserialize<TomlTable>(stream);
+
+        Assert.NotNull(table);
+        Assert.AreEqual(1L, (long)table!["a"]);
+    }
+}
+

src/Tomlyn/TomlSerializer.cs

@@ -19,6 +19,7 @@ public static class TomlSerializer
 
     private static readonly bool ReflectionEnabledByDefault = TomlSerializerFeatureSwitches.IsReflectionEnabledByDefaultCalculated;
     private static readonly Encoding DefaultStreamEncoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
+    private static ReadOnlySpan<byte> Utf8Bom => [0xEF, 0xBB, 0xBF];
 
     /// <summary>
     /// Gets a value indicating whether reflection-based serialization is enabled by default.
@@ -246,8 +247,11 @@ public static void Serialize(Stream utf8Stream, object? value, Type inputType, T
     public static T? Deserialize<T>(Stream utf8Stream, TomlSerializerOptions? options = null)
     {
         ArgumentGuard.ThrowIfNull(utf8Stream, nameof(utf8Stream));
-        using var reader = new StreamReader(utf8Stream, DefaultStreamEncoding, detectEncodingFromByteOrderMarks: true, bufferSize: 1024, leaveOpen: true);
-        return Deserialize<T>(reader, options);
+
+        var utf8Bytes = ReadUtf8Bytes(utf8Stream);
+        var effectiveOptions = options ?? TomlSerializerOptions.Default;
+        var typeInfo = ResolveTypeInfo(effectiveOptions, typeof(T));
+        return (T?)Deserialize(utf8Bytes, typeInfo);
     }
 
     /// <summary>
@@ -257,8 +261,8 @@ public static void Serialize(Stream utf8Stream, object? value, Type inputType, T
     {
         ArgumentGuard.ThrowIfNull(utf8Stream, nameof(utf8Stream));
         ArgumentGuard.ThrowIfNull(typeInfo, nameof(typeInfo));
-        using var reader = new StreamReader(utf8Stream, DefaultStreamEncoding, detectEncodingFromByteOrderMarks: true, bufferSize: 1024, leaveOpen: true);
-        return Deserialize<T>(reader, typeInfo);
+        var utf8Bytes = ReadUtf8Bytes(utf8Stream);
+        return (T?)Deserialize(utf8Bytes, (TomlTypeInfo)typeInfo);
     }
 
     /// <summary>
@@ -270,8 +274,11 @@ public static void Serialize(Stream utf8Stream, object? value, Type inputType, T
     {
         ArgumentGuard.ThrowIfNull(utf8Stream, nameof(utf8Stream));
         ArgumentGuard.ThrowIfNull(returnType, nameof(returnType));
-        using var reader = new StreamReader(utf8Stream, DefaultStreamEncoding, detectEncodingFromByteOrderMarks: true, bufferSize: 1024, leaveOpen: true);
-        return Deserialize(reader, returnType, options);
+
+        var utf8Bytes = ReadUtf8Bytes(utf8Stream);
+        var effectiveOptions = options ?? TomlSerializerOptions.Default;
+        var typeInfo = ResolveTypeInfo(effectiveOptions, returnType);
+        return Deserialize(utf8Bytes, typeInfo);
     }
 
     /// <summary>
@@ -325,6 +332,21 @@ public static bool TryDeserialize(string toml, Type returnType, out object? valu
 
         var options = typeInfo.Options;
         var reader = TomlReader.Create(toml, options);
+        return DeserializeCore(reader, typeInfo, options);
+    }
+
+    private static object? Deserialize(byte[] utf8Toml, TomlTypeInfo typeInfo)
+    {
+        ArgumentGuard.ThrowIfNull(utf8Toml, nameof(utf8Toml));
+        ArgumentGuard.ThrowIfNull(typeInfo, nameof(typeInfo));
+
+        var options = typeInfo.Options;
+        var reader = TomlReader.Create(utf8Toml, options);
+        return DeserializeCore(reader, typeInfo, options);
+    }
+
+    private static object? DeserializeCore(TomlReader reader, TomlTypeInfo typeInfo, TomlSerializerOptions options)
+    {
         reader.Read(); // StartDocument
         reader.Read(); // value start
 
@@ -359,6 +381,34 @@ public static bool TryDeserialize(string toml, Type returnType, out object? valu
         return typeInfo.ReadAsObject(reader);
     }
 
+    private static byte[] ReadUtf8Bytes(Stream utf8Stream)
+    {
+        if (utf8Stream is MemoryStream memoryStream && memoryStream.TryGetBuffer(out var buffer) &&
+            memoryStream.Position <= int.MaxValue && memoryStream.Length - memoryStream.Position <= int.MaxValue)
+        {
+            var start = buffer.Offset + (int)memoryStream.Position;
+            var length = (int)(memoryStream.Length - memoryStream.Position);
+            var data = new ReadOnlySpan<byte>(buffer.Array!, start, length);
+            return StripUtf8Bom(data).ToArray();
+        }
+
+        using var copy = new MemoryStream();
+        utf8Stream.CopyTo(copy);
+        var bytes = copy.ToArray();
+        var stripped = StripUtf8Bom(bytes);
+        return stripped.Length == bytes.Length ? bytes : stripped.ToArray();
+    }
+
+    private static ReadOnlySpan<byte> StripUtf8Bom(ReadOnlySpan<byte> data)
+    {
+        if (data.Length >= 3 && data[0] == Utf8Bom[0] && data[1] == Utf8Bom[1] && data[2] == Utf8Bom[2])
+        {
+            return data.Slice(3);
+        }
+
+        return data;
+    }
+
     /// <summary>
     /// Serializes a value to a writer using explicit metadata.
     /// </summary>