Fix scoped Utf8JsonReader to carry original position

src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Utf8JsonReader.cs

@@ -325,6 +325,9 @@ private static Utf8JsonReader GetReaderScopedToNextValue(ref Utf8JsonReader read
             ReadOnlySpan<byte> valueSpan = default;
             ReadOnlySequence<byte> valueSequence = default;
 
+            long lineNumber = 0;
+            long bytePositionInLine = 0;
+
             try
             {
                 switch (reader.TokenType)
@@ -345,6 +348,13 @@ private static Utf8JsonReader GetReaderScopedToNextValue(ref Utf8JsonReader read
                         }
                 }
 
+                // Capture the original reader's position so that the scoped reader, after consuming its
+                // first token, lands on the same position the original reader is currently at.
+                // The values captured here represent the position immediately after the current token,
+                // the per-case logic below rewinds them to the position immediately before the value token starts.
+                lineNumber = reader.CurrentState._lineNumber;
+                bytePositionInLine = reader.CurrentState._bytePositionInLine;
+
                 switch (reader.TokenType)
                 {
                     // Any of the "value start" states are acceptable.
@@ -371,6 +381,8 @@ private static Utf8JsonReader GetReaderScopedToNextValue(ref Utf8JsonReader read
                             valueSequence = sequence.Slice(startingOffset, totalLength);
                         }
 
+                        // Rewind by 1 byte to point right before the opening '{' or '['.
+                        bytePositionInLine--;
                         Debug.Assert(reader.TokenType is JsonTokenType.EndObject or JsonTokenType.EndArray);
                         break;
 
@@ -379,13 +391,16 @@ private static Utf8JsonReader GetReaderScopedToNextValue(ref Utf8JsonReader read
                     case JsonTokenType.True:
                     case JsonTokenType.False:
                     case JsonTokenType.Null:
+                        // Rewind by the length of the value token to point right before the start of the value.
                         if (reader.HasValueSequence)
                         {
                             valueSequence = reader.ValueSequence;
+                            bytePositionInLine -= valueSequence.Length;
                         }
                         else
                         {
                             valueSpan = reader.ValueSpan;
+                            bytePositionInLine -= valueSpan.Length;
                         }
 
                         break;
@@ -412,6 +427,9 @@ private static Utf8JsonReader GetReaderScopedToNextValue(ref Utf8JsonReader read
                                 $"Calculated span ends with {readerSpan[(int)reader.TokenStartIndex + payloadLength - 1]}");
 
                             valueSpan = readerSpan.Slice((int)reader.TokenStartIndex, payloadLength);
+
+                            // Rewind by payloadLength to point right before the opening quote.
+                            bytePositionInLine -= payloadLength;
                         }
                         else
                         {
@@ -427,6 +445,9 @@ private static Utf8JsonReader GetReaderScopedToNextValue(ref Utf8JsonReader read
                             Debug.Assert(
                                 valueSequence.ToArray()[payloadLength - 1] == (byte)'"',
                                 $"Calculated sequence ends with {valueSequence.ToArray()[payloadLength - 1]}");
+
+                            // Rewind by payloadLength to point right before the opening quote.
+                            bytePositionInLine -= payloadLength;
                         }
 
                         break;
@@ -451,10 +472,28 @@ private static Utf8JsonReader GetReaderScopedToNextValue(ref Utf8JsonReader read
             }
 
             Debug.Assert(!valueSpan.IsEmpty ^ !valueSequence.IsEmpty);
+            Debug.Assert(lineNumber >= 0);
+            Debug.Assert(bytePositionInLine >= 0);
+
+            // Carry only the position information and reader options to the scoped reader
+            // so that any JsonException it raises reports a position relative to the original input.
+            var scopedCurrentState = new JsonReaderState
+            (
+                lineNumber: lineNumber,
+                bytePositionInLine: bytePositionInLine,
+                inObject: default,
+                isNotPrimitive: default,
+                valueIsEscaped: default,
+                trailingCommaBeforeComment: default,
+                tokenType: default,
+                previousTokenType: default,
+                readerOptions: reader.CurrentState.Options,
+                bitStack: default
+            );
 
             return valueSpan.IsEmpty
-                ? new Utf8JsonReader(valueSequence, reader.CurrentState.Options)
-                : new Utf8JsonReader(valueSpan, reader.CurrentState.Options);
+                ? new Utf8JsonReader(valueSequence, isFinalBlock: true, state: scopedCurrentState)
+                : new Utf8JsonReader(valueSpan, isFinalBlock: true, state: scopedCurrentState);
         }
     }
 }

src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ReadValueTests.cs

@@ -781,6 +781,253 @@ public static void ReadSimpleList_AllowMultipleValues_TrailingContent()
             List<int> result = JsonSerializer.Deserialize<List<int>>(ref reader);
             Assert.Equal([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], result);
         }
+
+        [Fact]
+        public static void ReaderPreservesPositionInfo()
+        {
+            var utf8 = """
+                    [
+                        42
+                    ]
+                    """u8.ToArray();
+
+            JsonException ex = Assert.Throws<JsonException>(() =>
+            {
+                var reader = new Utf8JsonReader(utf8);
+
+                reader.Read();
+                reader.Read();
+
+                JsonSerializer.Deserialize<string>(ref reader);
+            });
+
+            Assert.Equal(1, ex.LineNumber);
+            Assert.Equal(6, ex.BytePositionInLine);
+        }
+
+        [Theory]
+        [InlineData("[  42]", typeof(string), 0, 5)]
+        [InlineData("[true]", typeof(string), 0, 5)]
+        [InlineData("[false]", typeof(string), 0, 6)]
+        [InlineData("[null]", typeof(int), 0, 5)]
+        [InlineData("[\"hello\"]", typeof(int), 0, 8)]
+        [InlineData("[{\"key\":1}]", typeof(string), 0, 2)]
+        [InlineData("[[1,2]]", typeof(string), 0, 2)]
+        public static void ReaderPreservesPositionInfoSingleLineTokens(
+            string json, Type deserializeType, long expectedLine, long expectedBytePosition)
+        {
+            byte[] utf8 = Encoding.UTF8.GetBytes(json);
+
+            JsonException ex = Assert.Throws<JsonException>(() =>
+            {
+                var reader = new Utf8JsonReader(utf8, isFinalBlock: true, state: default);
+                reader.Read();
+                reader.Read();
+
+                JsonSerializer.Deserialize(ref reader, deserializeType);
+            });
+
+            Assert.Equal(expectedLine, ex.LineNumber);
+            Assert.Equal(expectedBytePosition, ex.BytePositionInLine);
+        }
+
+        [Fact]
+        public static void ReaderPreservesPositionInfoNoneTokenType()
+        {
+            byte[] utf8 = "42"u8.ToArray();
+
+            JsonException ex = Assert.Throws<JsonException>(() =>
+            {
+                var reader = new Utf8JsonReader(utf8, isFinalBlock: true, state: default);
+
+                JsonSerializer.Deserialize<string>(ref reader);
+            });
+
+            Assert.Equal(0, ex.LineNumber);
+            Assert.Equal(2, ex.BytePositionInLine);
+        }
+
+        [Fact]
+        public static void ReaderPreservesPositionInfoPropertyNameTokenType()
+        {
+            byte[] utf8 = "{\"val\": 42}"u8.ToArray();
+
+            JsonException ex = Assert.Throws<JsonException>(() =>
+            {
+                var reader = new Utf8JsonReader(utf8, isFinalBlock: true, state: default);
+                reader.Read();
+                reader.Read();
+                Assert.Equal(JsonTokenType.PropertyName, reader.TokenType);
+
+                JsonSerializer.Deserialize<string>(ref reader);
+            });
+
+            Assert.Equal(0, ex.LineNumber);
+            Assert.Equal(10, ex.BytePositionInLine);
+        }
+
+        [Fact]
+        public static void ReaderPreservesPositionInfoPropertyNameMultiLine()
+        {
+            byte[] utf8 = Encoding.UTF8.GetBytes("{\n  \"val\":\n  42\n}");
+
+            JsonException ex = Assert.Throws<JsonException>(() =>
+            {
+                var reader = new Utf8JsonReader(utf8, isFinalBlock: true, state: default);
+                reader.Read();
+                reader.Read();
+                Assert.Equal(JsonTokenType.PropertyName, reader.TokenType);
+
+                JsonSerializer.Deserialize<string>(ref reader);
+            });
+
+            Assert.Equal(2, ex.LineNumber);
+            Assert.Equal(4, ex.BytePositionInLine);
+        }
+
+        [Theory]
+        [InlineData("[1234]", 2, typeof(string), 0, 5)]
+        [InlineData("[true]", 3, typeof(string), 0, 5)]
+        [InlineData("[\"hello\"]", 4, typeof(int), 0, 8)]
+        [InlineData("[{\"key\":1}]", 5, typeof(string), 0, 2)]
+        public static void ReaderPreservesPositionInfoMultiSegment(string json, int splitAt, Type deserializeType, long expectedLine, long expectedBytePosition)
+        {
+            byte[] utf8 = Encoding.UTF8.GetBytes(json);
+            ReadOnlySequence<byte> sequence = JsonTestHelper.CreateSegments(utf8, splitAt);
+
+            JsonException ex = Assert.Throws<JsonException>(() =>
+            {
+                var reader = new Utf8JsonReader(sequence, isFinalBlock: true, state: default);
+                reader.Read();
+                reader.Read();
+
+                JsonSerializer.Deserialize(ref reader, deserializeType);
+            });
+
+            Assert.Equal(expectedLine, ex.LineNumber);
+            Assert.Equal(expectedBytePosition, ex.BytePositionInLine);
+        }
+
+        [Fact]
+        public static void ReaderPreservesPositionInfoMultiByteUtf8String()
+        {
+            // "😀葛🀄" occupies 11 bytes in UTF-8 (4 + 3 + 4),
+            // so the closing quote sits at byte index 13 and BytePositionInLine after the token is 14.
+            byte[] utf8 = Encoding.UTF8.GetBytes("[\"😀葛🀄\"]");
+
+            JsonException ex = Assert.Throws<JsonException>(() =>
+            {
+                var reader = new Utf8JsonReader(utf8, isFinalBlock: true, state: default);
+                reader.Read();
+                reader.Read();
+                Assert.Equal(JsonTokenType.String, reader.TokenType);
+
+                JsonSerializer.Deserialize<int>(ref reader);
+            });
+
+            Assert.Equal(0, ex.LineNumber);
+            Assert.Equal(14, ex.BytePositionInLine);
+        }
+
+        [Theory]
+        [InlineData("[\n  {\"key\":1}\n]", typeof(string), 1, 3)]
+        [InlineData("[\n  [1, 2]\n]", typeof(string), 1, 3)]
+        public static void ReaderPreservesPositionInfoMultiLineContainer(
+            string json, Type deserializeType, long expectedLine, long expectedBytePosition)
+        {
+            byte[] utf8 = Encoding.UTF8.GetBytes(json);
+
+            JsonException ex = Assert.Throws<JsonException>(() =>
+            {
+                var reader = new Utf8JsonReader(utf8, isFinalBlock: true, state: default);
+                reader.Read();
+                reader.Read();
+                Assert.True(reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray);
+
+                JsonSerializer.Deserialize(ref reader, deserializeType);
+            });
+
+            Assert.Equal(expectedLine, ex.LineNumber);
+            Assert.Equal(expectedBytePosition, ex.BytePositionInLine);
+        }
+
+        [Theory]
+        [InlineData("[ /* comment */ 42 ]", typeof(string), 0, 18)]
+        [InlineData("[ // comment\n42 ]", typeof(string), 1, 2)]
+        [InlineData("[ /* comment */ true ]", typeof(int), 0, 20)]
+        [InlineData("[ /* comment */ false ]", typeof(int), 0, 21)]
+        [InlineData("[ /* comment */ null ]", typeof(int), 0, 20)]
+        [InlineData("[ /* comment */ \"hello\" ]", typeof(int), 0, 23)]
+        [InlineData("[ /* comment */ {\"key\":1} ]", typeof(string), 0, 17)]
+        [InlineData("[ /* comment */ [1,2] ]", typeof(string), 0, 17)]
+        [InlineData("[ /*\nmultiline\ncomment\n*/ 42 ]", typeof(string), 3, 5)]
+        [InlineData("[ /*\n*/ 42 ]", typeof(string), 1, 5)]
+        [InlineData("[ /*\nmultiline\n*/ {\"key\":1} ]", typeof(string), 2, 4)]
+        [InlineData("[ /*\nmultiline\n*/ [1,2] ]", typeof(string), 2, 4)]
+        [InlineData("[ /*\nmultiline\n*/ \"hello\" ]", typeof(int), 2, 10)]
+        public static void ReaderPreservesPositionInfoWithSkippedComments(
+            string json, Type deserializeType, long expectedLine, long expectedBytePosition)
+        {
+            byte[] utf8 = Encoding.UTF8.GetBytes(json);
+            var options = new JsonReaderOptions { CommentHandling = JsonCommentHandling.Skip };
+
+            JsonException ex = Assert.Throws<JsonException>(() =>
+            {
+                var reader = new Utf8JsonReader(utf8, isFinalBlock: true, state: new JsonReaderState(options));
+                reader.Read();
+                reader.Read();
+
+                JsonSerializer.Deserialize(ref reader, deserializeType);
+            });
+
+            Assert.Equal(expectedLine, ex.LineNumber);
+            Assert.Equal(expectedBytePosition, ex.BytePositionInLine);
+        }
+
+        [Theory]
+        [InlineData("{\"val\": /* comment */ 42}", 0, 24)]
+        [InlineData("{\"val\": // comment\n42}", 1, 2)]
+        [InlineData("{\"val\":\n/* comment */\n42}", 2, 2)]
+        [InlineData("{\"val\": /* comment */ {\"k\":1}}", 0, 23)]
+        public static void ReaderPreservesPositionInfoPropertyNameWithSkippedComments(
+            string json, long expectedLine, long expectedBytePosition)
+        {
+            byte[] utf8 = Encoding.UTF8.GetBytes(json);
+            var options = new JsonReaderOptions { CommentHandling = JsonCommentHandling.Skip };
+
+            JsonException ex = Assert.Throws<JsonException>(() =>
+            {
+                var reader = new Utf8JsonReader(utf8, isFinalBlock: true, state: new JsonReaderState(options));
+                reader.Read();
+                reader.Read();
+                Assert.Equal(JsonTokenType.PropertyName, reader.TokenType);
+
+                JsonSerializer.Deserialize<string>(ref reader);
+            });
+
+            Assert.Equal(expectedLine, ex.LineNumber);
+            Assert.Equal(expectedBytePosition, ex.BytePositionInLine);
+        }
+
+        [Theory]
+        [InlineData("/* comment */ 42", 0, 16)]
+        [InlineData("/*\ncomment\n*/ 42", 2, 5)]
+        public static void ReaderPreservesPositionInfoWithCommentBeforeNoneToken(
+            string json, long expectedLine, long expectedBytePosition)
+        {
+            byte[] utf8 = Encoding.UTF8.GetBytes(json);
+            var options = new JsonReaderOptions { CommentHandling = JsonCommentHandling.Skip };
+
+            JsonException ex = Assert.Throws<JsonException>(() =>
+            {
+                var reader = new Utf8JsonReader(utf8, isFinalBlock: true, state: new JsonReaderState(options));
+
+                JsonSerializer.Deserialize<string>(ref reader);
+            });
+
+            Assert.Equal(expectedLine, ex.LineNumber);
+            Assert.Equal(expectedBytePosition, ex.BytePositionInLine);
+        }
     }
 
     // From https://github.com/dotnet/runtime/issues/882