Started testing JSON lexer.

Directory.Build.props

@@ -56,4 +56,11 @@
         </None>
     </ItemGroup>
 
+    <ItemGroup>
+        <PackageReference Include="PolySharp" Version="1.14.1">
+            <PrivateAssets>all</PrivateAssets>
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
+    </ItemGroup>
+
 </Project>

src/Pasper.Json/Parsing/JsonLexer.cs

@@ -43,7 +43,7 @@ public void Reset()
 
     private bool TryGetNextToken([NotNullWhen(true)] out IToken? token)
     {
-        SkipWhitespace();
+        SkipIgnoredCharacters();
         if (_currentIndex >= json.Length)
         {
             token = null;
@@ -69,16 +69,23 @@ private bool TryGetNextToken([NotNullWhen(true)] out IToken? token)
         throw new UnexpectedTokenException(_lineNumber, _columnNumber, currentCharacter.ToString());
     }
 
-    private void SkipWhitespace()
+    private void SkipIgnoredCharacters()
     {
         while (_currentIndex < json.Length)
         {
-            var currentChar = json[_currentIndex++];
+            var currentChar = json[_currentIndex];
+            if (currentChar is ':' or ',')
+            {
+                _currentIndex++;
+                continue;
+            }
+
             if (!char.IsWhiteSpace(currentChar))
                 break;
 
             if (currentChar != '\n' && currentChar != '\r')
             {
+                _currentIndex++;
                 _columnNumber++;
                 continue;
             }
@@ -88,6 +95,7 @@ private void SkipWhitespace()
                 _currentIndex++;
 
             _lineNumber++;
+            _currentIndex++;
             _columnNumber = 0;
         }
     }

test/Pasper.Json.Tests/Parsing/JsonLexer/MoveNext.cs

@@ -0,0 +1,62 @@
+using Pasper.Json.Parsing;
+using Pasper.Json.Tokens;
+
+namespace Pasper.Json.Tests.Parsing;
+
+public sealed partial class JsonLexerTests
+{
+    [Fact]
+    public void MoveNext_ShouldMoveToNextToken()
+    {
+        var lexer = new JsonLexer("{}");
+        var tokenAvailable = lexer.MoveNext();
+        Assert.True(tokenAvailable);
+        Assert.NotNull(lexer.Current);
+        Assert.IsType<BeginObjectToken>(lexer.Current);
+
+        tokenAvailable = lexer.MoveNext();
+        Assert.True(tokenAvailable);
+        Assert.NotNull(lexer.Current);
+        Assert.IsType<EndObjectToken>(lexer.Current);
+    }
+
+    [Fact]
+    public void MoveNext_ShouldReturnFalseWhenNoMoreTokens()
+    {
+        var lexer = new JsonLexer("{}");
+        var tokenAvailable = lexer.MoveNext();
+        Assert.True(tokenAvailable);
+        Assert.NotNull(lexer.Current);
+
+        tokenAvailable = lexer.MoveNext();
+        Assert.True(tokenAvailable);
+        Assert.NotNull(lexer.Current);
+
+        tokenAvailable = lexer.MoveNext();
+        Assert.False(tokenAvailable);
+    }
+
+    [Fact]
+    public void MoveNext_ShouldReturnFalseWhenNoTokens()
+    {
+        var lexer = new JsonLexer("");
+        var tokenAvailable = lexer.MoveNext();
+        Assert.False(tokenAvailable);
+    }
+
+    [Fact]
+    public void MoveNext_ShouldSetPreviousToken()
+    {
+        var lexer = new JsonLexer("{}");
+        var tokenAvailable = lexer.MoveNext();
+        Assert.True(tokenAvailable);
+        Assert.NotNull(lexer.Current);
+        Assert.Null(lexer.Previous);
+
+        tokenAvailable = lexer.MoveNext();
+        Assert.True(tokenAvailable);
+        Assert.NotNull(lexer.Current);
+        Assert.NotNull(lexer.Previous);
+        Assert.IsType<BeginObjectToken>(lexer.Previous);
+    }
+}

test/Pasper.Json.Tests/Parsing/JsonLexer/Reset.cs

@@ -0,0 +1,25 @@
+using Pasper.Json.Parsing;
+
+namespace Pasper.Json.Tests.Parsing;
+
+public sealed partial class JsonLexerTests
+{
+    [Fact]
+    public void Reset_ShouldResetLexer()
+    {
+        var lexer = new JsonLexer("{}");
+        var tokenAvailable = lexer.MoveNext();
+        Assert.True(tokenAvailable);
+        Assert.NotNull(lexer.Current);
+
+        tokenAvailable = lexer.MoveNext();
+        Assert.True(tokenAvailable);
+        Assert.NotNull(lexer.Current);
+        Assert.NotNull(lexer.Previous);
+        Assert.NotEqual(lexer.Current, lexer.Previous);
+
+        lexer.Reset();
+        Assert.Null(lexer.Current);
+        Assert.Null(lexer.Previous);
+    }
+}

test/Pasper.Json.Tests/Parsing/JsonLexer/SkipIgnoredCharacters.cs

@@ -0,0 +1,22 @@
+using Pasper.Json.Parsing;
+
+namespace Pasper.Json.Tests.Parsing;
+
+public sealed partial class JsonLexerTests
+{
+    [Theory]
+    [InlineData(" ")]
+    [InlineData("\t")]
+    [InlineData("\n")]
+    [InlineData("\r")]
+    [InlineData("\r\n")]
+    [InlineData(":")]
+    [InlineData(",")]
+    public void SkipIgnoredCharacters(string input)
+    {
+        var lexer = new JsonLexer(input);
+        var tokenAvailable = lexer.MoveNext();
+        Assert.False(tokenAvailable);
+        Assert.Null(lexer.Current);
+    }
+}