allowing thousand separators + fix the consumed indicator

.github/workflows/dotnet.yml

@@ -24,6 +24,11 @@ jobs:
           8.0.x
           9.0.x
 
+    - name: Install Mono (for .NET Framework 4.7.2 test host)
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mono-complete
+
     - name: Restore dependencies
       run: dotnet restore
 

README.md

@@ -95,6 +95,33 @@ For UTF-8 or ASCII inputs, you may pass a `ReadOnlySpan<byte>` argument. You can
 an optional `out int characters_consumed` parameter to track the number of characters consumed
 by the number pattern.
 
+## Thousands separators
+
+To match the behaviour of `Double.Parse` / `Single.Parse` on inputs such as `"1,234,567.89"`,
+pass `NumberStyles.AllowThousands`. The separator defaults to `,` and can be overridden via
+the optional `thousands_separator` parameter (e.g. for the European convention where the
+decimal separator is `,` and the thousands separator is `.`):
+
+```C#
+using System.Globalization;
+using csFastFloat;
+
+// Invariant culture: "," as thousands, "." as decimal.
+double a = FastDoubleParser.ParseDouble(
+    "1,234,567.89",
+    NumberStyles.Float | NumberStyles.AllowThousands);                // 1234567.89
+
+// European convention: "." as thousands, "," as decimal.
+double b = FastDoubleParser.ParseDouble(
+    "1.234.567,89",
+    NumberStyles.Float | NumberStyles.AllowThousands,
+    decimal_separator: ',',
+    thousands_separator: '.');                                        // 1234567.89
+```
+
+Without `AllowThousands`, the parser stops at the first separator (so `"1,234"` parses as
+`1` with `characters_consumed == 1`), preserving the prior default behaviour.
+
 
 
 

TestcsFastFloat/Basic/TestDoubleParser.cs

@@ -291,6 +291,107 @@ unsafe public void cas_compute_float_64_2()
       }
     }
 
+    [Theory]
+    [InlineData("  1", 3)]
+    [InlineData("   -1.5", 7)]
+    [InlineData(" \t 2.5e3 trailing", 8)]
+    [InlineData("1.5", 3)]
+    public void TryParseDouble_String_CharsConsumed_IncludesLeadingWhitespace(string input, int expectedConsumed)
+    {
+      Assert.True(FastDoubleParser.TryParseDouble(input, out int consumed, out double _));
+      Assert.Equal(expectedConsumed, consumed);
+    }
+
+    [Theory]
+    [InlineData("  1", 3)]
+    [InlineData("   -1.5", 7)]
+    [InlineData(" \t 2.5e3 trailing", 8)]
+    public unsafe void TryParseDouble_CharPointer_CharsConsumed_IncludesLeadingWhitespace(string input, int expectedConsumed)
+    {
+      fixed (char* p = input)
+      {
+        Assert.True(FastDoubleParser.TryParseDouble(p, p + input.Length, out int consumed, out double _));
+        Assert.Equal(expectedConsumed, consumed);
+      }
+    }
+
+    [Theory]
+    [InlineData("  1", 3)]
+    [InlineData("   -1.5", 7)]
+    [InlineData(" \t 2.5e3 trailing", 8)]
+    public unsafe void TryParseDouble_BytePointer_CharsConsumed_IncludesLeadingWhitespace(string input, int expectedConsumed)
+    {
+      byte[] bytes = Encoding.ASCII.GetBytes(input);
+      fixed (byte* p = bytes)
+      {
+        Assert.True(FastDoubleParser.TryParseDouble(p, p + bytes.Length, out int consumed, out double _));
+        Assert.Equal(expectedConsumed, consumed);
+      }
+    }
+
+    [Fact]
+    public void TryParseDouble_ConsumedCount_AdvancesPastWholeInput()
+    {
+      string sut = "  1.5 -2.25e1 7";
+      int pos = 0;
+      int parsed = 0;
+      while (pos < sut.Length
+             && FastDoubleParser.TryParseDouble(sut.AsSpan(pos), out int consumed, out double _))
+      {
+        Assert.True(consumed > 0);
+        pos += consumed;
+        parsed++;
+      }
+      Assert.Equal(3, parsed);
+      Assert.Equal(sut.Length, pos);
+    }
+
+    [Theory]
+    [InlineData("1,234,567.89", 1234567.89)]
+    [InlineData("1,234", 1234.0)]
+    [InlineData("12,345.5", 12345.5)]
+    [InlineData("-1,234.5", -1234.5)]
+    [InlineData("1,234,567,890,123,456,789,012,345", 1234567890123456789012345d)]
+    public void ParseDouble_AllowThousands_MatchesBcl(string input, double expected)
+    {
+      var styles = NumberStyles.Float | NumberStyles.AllowThousands;
+      var bcl = double.Parse(input, styles, CultureInfo.InvariantCulture);
+      Assert.Equal(expected, bcl);
+
+      var ff = FastDoubleParser.ParseDouble(input, styles);
+      Assert.Equal(bcl, ff);
+    }
+
+    [Fact]
+    public void ParseDouble_NoAllowThousands_StopsAtSeparator()
+    {
+      Assert.True(FastDoubleParser.TryParseDouble("1,234.5", out int consumed, out double result));
+      Assert.Equal(1d, result);
+      Assert.Equal(1, consumed);
+    }
+
+    [Fact]
+    public void ParseDouble_AllowThousands_CustomSeparator()
+    {
+      // European convention: '.' as thousands, ',' as decimal.
+      var styles = NumberStyles.Float | NumberStyles.AllowThousands;
+      var ff = FastDoubleParser.ParseDouble("1.234.567,89", styles, decimal_separator: ',', thousands_separator: '.');
+      Assert.Equal(1234567.89, ff);
+    }
+
+    [Fact]
+    public unsafe void ParseDouble_AllowThousands_BytePointer()
+    {
+      byte[] bytes = Encoding.ASCII.GetBytes("1,234,567.89");
+      var styles = NumberStyles.Float | NumberStyles.AllowThousands;
+      fixed (byte* p = bytes)
+      {
+        Assert.True(FastDoubleParser.TryParseDouble(p, p + bytes.Length, out int consumed, out double result, styles));
+        Assert.Equal(1234567.89, result);
+        Assert.Equal(bytes.Length, consumed);
+      }
+    }
+
     private static double[] testing_power_of_ten =  {
     1e-307, 1e-306, 1e-305, 1e-304, 1e-303, 1e-302, 1e-301, 1e-300, 1e-299,
     1e-298, 1e-297, 1e-296, 1e-295, 1e-294, 1e-293, 1e-292, 1e-291, 1e-290,

TestcsFastFloat/Basic/TestFloatParser.cs

@@ -2,6 +2,7 @@
 using csFastFloat.Structures;
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Text;
 using Xunit;
 
@@ -260,6 +261,126 @@ unsafe public void ParseNumber_Works_Scenarios()
       }
     }
 
+    [Theory]
+    [InlineData("  1", 3)]
+    [InlineData("   -1.5", 7)]
+    [InlineData(" \t 2.5e3 trailing", 8)]
+    [InlineData("1.5", 3)]
+    public void TryParseFloat_String_CharsConsumed_IncludesLeadingWhitespace(string input, int expectedConsumed)
+    {
+      Assert.True(FastFloatParser.TryParseFloat(input, out int consumed, out float _));
+      Assert.Equal(expectedConsumed, consumed);
+    }
+
+    [Theory]
+    [InlineData("  1", 3)]
+    [InlineData("   -1.5", 7)]
+    [InlineData(" \t 2.5e3 trailing", 8)]
+    public unsafe void TryParseFloat_CharPointer_CharsConsumed_IncludesLeadingWhitespace(string input, int expectedConsumed)
+    {
+      fixed (char* p = input)
+      {
+        Assert.True(FastFloatParser.TryParseFloat(p, p + input.Length, out int consumed, out float _));
+        Assert.Equal(expectedConsumed, consumed);
+      }
+    }
+
+    [Theory]
+    [InlineData("  1", 3)]
+    [InlineData("   -1.5", 7)]
+    [InlineData(" \t 2.5e3 trailing", 8)]
+    public unsafe void TryParseFloat_BytePointer_CharsConsumed_IncludesLeadingWhitespace(string input, int expectedConsumed)
+    {
+      byte[] bytes = Encoding.ASCII.GetBytes(input);
+      fixed (byte* p = bytes)
+      {
+        Assert.True(FastFloatParser.TryParseFloat(p, p + bytes.Length, out int consumed, out float _));
+        Assert.Equal(expectedConsumed, consumed);
+      }
+    }
+
+    [Fact]
+    public void TryParseFloat_ConsumedCount_AdvancesPastWholeInput()
+    {
+      // Walking the whole string using consumed count should land exactly at the end.
+      string sut = "  1.5 -2.25e1 7";
+      int pos = 0;
+      int parsed = 0;
+      while (pos < sut.Length
+             && FastFloatParser.TryParseFloat(sut.AsSpan(pos), out int consumed, out float _))
+      {
+        Assert.True(consumed > 0);
+        pos += consumed;
+        parsed++;
+      }
+      Assert.Equal(3, parsed);
+      Assert.Equal(sut.Length, pos);
+    }
+
+    [Theory]
+    [InlineData("1,234,567.89", 1234567.89f)]
+    [InlineData("1,234", 1234f)]
+    [InlineData("12,345.5", 12345.5f)]
+    [InlineData("-1,234.5", -1234.5f)]
+    public void ParseFloat_AllowThousands_MatchesBcl(string input, float expected)
+    {
+      var styles = NumberStyles.Float | NumberStyles.AllowThousands;
+      var bcl = float.Parse(input, styles, CultureInfo.InvariantCulture);
+      Assert.Equal(expected, bcl);
+
+      var ff = FastFloatParser.ParseFloat(input, styles);
+      Assert.Equal(bcl, ff);
+    }
+
+    [Fact]
+    public void ParseFloat_AllowThousands_FromPR110()
+    {
+      // The exact case from PR #110.
+      var styles = NumberStyles.Float | NumberStyles.AllowThousands;
+      var bcl = float.Parse("1,234,567.89", styles, CultureInfo.InvariantCulture);
+      var ff = FastFloatParser.ParseFloat("1,234,567.89", styles);
+      Assert.Equal(bcl, ff);
+    }
+
+    [Fact]
+    public void ParseFloat_NoAllowThousands_StopsAtSeparator()
+    {
+      // Without AllowThousands, the comma is not part of the number — parsing stops at it.
+      Assert.True(FastFloatParser.TryParseFloat("1,234.5", out int consumed, out float result));
+      Assert.Equal(1f, result);
+      Assert.Equal(1, consumed);
+    }
+
+    [Fact]
+    public void ParseFloat_AllowThousands_CustomSeparator()
+    {
+      // European convention: '.' as thousands, ',' as decimal.
+      var styles = NumberStyles.Float | NumberStyles.AllowThousands;
+      var ff = FastFloatParser.ParseFloat("1.234.567,89", styles, decimal_separator: ',', thousands_separator: '.');
+      Assert.Equal(1234567.89f, ff);
+    }
+
+    [Fact]
+    public void ParseFloat_AllowThousands_RejectsLeadingSeparator()
+    {
+      // BCL rejects a leading comma even with AllowThousands; we should too.
+      var styles = NumberStyles.Float | NumberStyles.AllowThousands;
+      Assert.Throws<FormatException>(() => float.Parse(",234", styles, CultureInfo.InvariantCulture));
+      Assert.False(FastFloatParser.TryParseFloat(",234", out _, out float _, styles));
+    }
+
+    [Fact]
+    public void ParseFloat_AllowThousands_RequiresSeparatorBetweenDigits()
+    {
+      // We require thousands separators to lie strictly between digits; "1,,234" stops
+      // parsing at the first comma (since the next char is not a digit), so we consume
+      // only "1". This is intentionally stricter than the very permissive BCL behavior.
+      var styles = NumberStyles.Float | NumberStyles.AllowThousands;
+      Assert.True(FastFloatParser.TryParseFloat("1,,234", out int consumed, out float result, styles));
+      Assert.Equal(1f, result);
+      Assert.Equal(1, consumed);
+    }
+
     private static float[] testing_power_of_ten_float =  {
     1e-65F,
     1e-64F,  1e-63F,  1e-62F,  1e-61F,  1e-60F,  1e-59F,  1e-58F,  1e-57F,  1e-56F,