diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..c4ec1a1 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,47 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "IntCalc", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "build", + "program": "${workspaceFolder}/sample/IntCalc/bin/Debug/netcoreapp2.0/IntCalc.dll", + "args": [ "IntCalc" ], + "cwd": "${workspaceFolder}/sample/IntCalc", + "console": "integratedTerminal", + "stopAtEntry": false, + "internalConsoleOptions": "openOnSessionStart" + }, + { + "name": "JsonParser", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "build", + "program": "${workspaceFolder}/sample/JsonParser/bin/Debug/netcoreapp2.0/JsonParser.dll", + "args": [ "IntCalc" ], + "cwd": "${workspaceFolder}/sample/JsonParser", + "console": "integratedTerminal", + "stopAtEntry": false, + "internalConsoleOptions": "openOnSessionStart" + }, + { + "name": "DateTimeTextParser", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "build", + "program": "${workspaceFolder}/sample/DateTimeTextParser/bin/Debug/netcoreapp2.0/DateTimeParser.dll", + "args": [ "IntCalc" ], + "cwd": "${workspaceFolder}/sample/DateTimeTextParser", + "console": "integratedTerminal", + "stopAtEntry": false, + "internalConsoleOptions": "openOnSessionStart" + }, + { + "name": ".NET Core Attach", + "type": "coreclr", + "request": "attach", + "processId": "${command:pickProcess}" + } + ] +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..9120d06 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,34 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "build", + "command": "dotnet", + "type": "process", + "group": { + "kind": "build", + "isDefault": true + }, + "args": [ + "build", + "/p:GenerateFullPaths=true" + ], + "problemMatcher": "$msCompile" + }, + { + "label": "test", + "command": "dotnet", + "type": "process", + "group": { + "kind": "test", + "isDefault": true + }, + "args": [ + "test", + "${workspaceFolder}/test/Superpower.Tests/Superpower.Tests.csproj", + "/p:GenerateFullPaths=true" + ], + "problemMatcher": "$msCompile" + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md index 2acaab4..06e2fd7 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,7 @@ Superpower is introduced, with a worked example, in [this blog post](https://nbl * [_DateTimeTextParser_](https://github.com/datalust/superpower/tree/dev/sample/DateTimeTextParser) shows how Superpower's text parsers work, parsing ISO-8601 date-times * [_IntCalc_](https://github.com/datalust/superpower/tree/dev/sample/IntCalc) is a simple arithmetic expresion parser (`1 + 2 * 3`) included in the repository, demonstrating how Superpower token parsing works * [_Plotty_](https://github.com/SuperJMN/Plotty) implements an instruction set for a RISC virtual machine +* [_tcalc_](https://github.com/nblumhardt/tcalc) is an example expression language that computes durations (`1d / 12m`) **Real-world** projects built with Superpower: diff --git a/appveyor.yml b/appveyor.yml index bb1c536..2172a9b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,7 +10,7 @@ artifacts: deploy: - provider: NuGet api_key: - secure: 2APsxl7cOn94Y5lvJ0LshnVGefMpKu/JUG6IiM6BK9UWq1bN0Zv4X4OBXzkMPFm0 + secure: Xi+qouQ6+cOJ85PMOQKEGy+MC1EKyYJCoF2YfuE9Rcj6lvNtm08TIZllTJCqId+M skip_symbols: true on: branch: /^(master|dev)$/ diff --git a/sample/IntCalc/ArithmeticExpressionTokenizer.cs b/sample/IntCalc/ArithmeticExpressionTokenizer.cs index 59d9076..3569405 100644 --- a/sample/IntCalc/ArithmeticExpressionTokenizer.cs +++ b/sample/IntCalc/ArithmeticExpressionTokenizer.cs @@ -27,13 +27,14 @@ protected override IEnumerable> Tokenize(TextS { ArithmeticExpressionToken charToken; - if (char.IsDigit(next.Value)) + var ch = next.Value; + if (ch >= '0' && ch <= '9') { var integer = Numerics.Integer(next.Location); next = integer.Remainder.ConsumeChar(); yield return Result.Value(ArithmeticExpressionToken.Number, integer.Location, integer.Remainder); } - else if (_operators.TryGetValue(next.Value, out charToken)) + else if (_operators.TryGetValue(ch, out charToken)) { yield return Result.Value(charToken, next.Location, next.Remainder); next = next.Remainder.ConsumeChar(); diff --git a/sample/IntCalc/IntCalc.csproj b/sample/IntCalc/IntCalc.csproj index ea1e06a..0ab3707 100644 --- a/sample/IntCalc/IntCalc.csproj +++ b/sample/IntCalc/IntCalc.csproj @@ -1,7 +1,12 @@  + + netcoreapp1.0 + + + net46;netcoreapp2.0 + - net46;netcoreapp1.0 IntCalc Exe IntCalc diff --git a/src/Superpower/Combinators.cs b/src/Superpower/Combinators.cs index 56e3907..ecf8317 100644 --- a/src/Superpower/Combinators.cs +++ b/src/Superpower/Combinators.cs @@ -410,7 +410,7 @@ public static TokenListParser Many(this TokenListParser(r); - return TokenListParserResult.Value(result.ToArray(), input, r.Remainder); + return TokenListParserResult.Value(result.ToArray(), input, from); }; } @@ -444,7 +444,7 @@ public static TextParser Many(this TextParser parser) if (!r.Backtrack && r.IsPartial(from)) return Result.CastEmpty(r); - return Result.Value(result.ToArray(), input, r.Remainder); + return Result.Value(result.ToArray(), input, from); }; } @@ -476,7 +476,7 @@ public static TextParser IgnoreMany(this TextParser parser) if (!r.Backtrack && r.IsPartial(from)) return Result.CastEmpty(r); - return Result.Value(Unit.Value, input, r.Remainder); + return Result.Value(Unit.Value, input, from); }; } diff --git a/src/Superpower/Display/Presentation.cs b/src/Superpower/Display/Presentation.cs index 8e7a2f9..fabe80d 100644 --- a/src/Superpower/Display/Presentation.cs +++ b/src/Superpower/Display/Presentation.cs @@ -69,26 +69,76 @@ public static string FormatAppearance(TKind kind, string value) return $"{FormatKind(kind)} {clipped}"; } - public static string FormatLiteral(char literal) { switch (literal) { - case '\r': - return "carriage return"; - case '\n': - return "line feed"; - case '\t': - return "tab"; - case '\0': - return "NUL"; - default: - return "`" + literal + "`"; + //Unicode Category: Space Separators + case '\x00A0': return "U+00A0 no-break space"; + case '\x1680': return "U+1680 ogham space mark"; + case '\x2000': return "U+2000 en quad"; + case '\x2001': return "U+2001 em quad"; + case '\x2002': return "U+2002 en space"; + case '\x2003': return "U+2003 em space"; + case '\x2004': return "U+2004 three-per-em space"; + case '\x2005': return "U+2005 four-per-em space"; + case '\x2006': return "U+2006 six-per-em space"; + case '\x2007': return "U+2007 figure space"; + case '\x2008': return "U+2008 punctuation space"; + case '\x2009': return "U+2009 thin space"; + case '\x200A': return "U+200A hair space"; + case '\x202F': return "U+202F narrow no-break space"; + case '\x205F': return "U+205F medium mathematical space"; + case '\x3000': return "U+3000 ideographic space"; + + //Line Separator + case '\x2028': return "U+2028 line separator"; + + //Paragraph Separator + case '\x2029': return "U+2029 paragraph separator"; + + //Unicode C0 Control Codes (ASCII equivalent) + case '\x0000': return "NUL"; //\0 + case '\x0001': return "U+0001 start of heading"; + case '\x0002': return "U+0002 start of text"; + case '\x0003': return "U+0003 end of text"; + case '\x0004': return "U+0004 end of transmission"; + case '\x0005': return "U+0005 enquiry"; + case '\x0006': return "U+0006 acknowledge"; + case '\x0007': return "U+0007 bell"; + case '\x0008': return "U+0008 backspace"; + case '\x0009': return "tab"; //\t + case '\x000A': return "line feed"; //\n + case '\x000B': return "U+000B vertical tab"; + case '\x000C': return "U+000C form feed"; + case '\x000D': return "carriage return"; //\r + case '\x000E': return "U+000E shift in"; + case '\x000F': return "U+000F shift out"; + case '\x0010': return "U+0010 data link escape"; + case '\x0011': return "U+0011 device ctrl 1"; + case '\x0012': return "U+0012 device ctrl 2"; + case '\x0013': return "U+0013 device ctrl 3"; + case '\x0014': return "U+0014 device ctrl 4"; + case '\x0015': return "U+0015 not acknowledge"; + case '\x0016': return "U+0016 synchronous idle"; + case '\x0017': return "U+0017 end transmission block"; + case '\x0018': return "U+0018 cancel"; + case '\x0019': return "U+0019 end of medium"; + case '\x0020': return "space"; + case '\x001A': return "U+001A substitute"; + case '\x001B': return "U+001B escape"; + case '\x001C': return "U+001C file separator"; + case '\x001D': return "U+001D group separator"; + case '\x001E': return "U+001E record separator"; + case '\x001F': return "U+001F unit separator"; + case '\x007F': return "U+007F delete"; + + default: return "`" + literal + "`"; } } public static string FormatLiteral(string literal) - { + { return "`" + literal + "`"; } } diff --git a/src/Superpower/Model/Result.cs b/src/Superpower/Model/Result.cs index 88d68b0..ab3c36c 100644 --- a/src/Superpower/Model/Result.cs +++ b/src/Superpower/Model/Result.cs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +using Superpower.Util; + namespace Superpower.Model { /// @@ -95,14 +97,7 @@ public static Result CombineEmpty(Result first, Result second) if (expectations == null) expectations = second.Expectations; else if (second.Expectations != null) - { - expectations = new string[first.Expectations.Length + second.Expectations.Length]; - var i = 0; - for (; i < first.Expectations.Length; ++i) - expectations[i] = first.Expectations[i]; - for (var j = 0; j < second.Expectations.Length; ++i, ++j) - expectations[i] = second.Expectations[j]; - } + expectations = ArrayEnumerable.Concat(first.Expectations, second.Expectations); return new Result(second.Remainder, second.ErrorMessage, expectations, second.Backtrack); } diff --git a/src/Superpower/Model/Result`1.cs b/src/Superpower/Model/Result`1.cs index d6d0a3e..36cb0b0 100644 --- a/src/Superpower/Model/Result`1.cs +++ b/src/Superpower/Model/Result`1.cs @@ -68,7 +68,7 @@ public T Value get { if (!HasValue) - throw new InvalidOperationException("Result has no value."); + throw new InvalidOperationException($"{nameof(Result)} has no value."); return _value; } } @@ -141,7 +141,7 @@ public string FormatErrorMessageFragment() else { var next = Location.ConsumeChar().Value; - message = $"unexpected `{next}`"; + message = $"unexpected {Display.Presentation.FormatLiteral(next)}"; } if (Expectations != null) diff --git a/src/Superpower/Model/TextSpan.cs b/src/Superpower/Model/TextSpan.cs index bd4311c..e94b556 100644 --- a/src/Superpower/Model/TextSpan.cs +++ b/src/Superpower/Model/TextSpan.cs @@ -128,13 +128,16 @@ public override int GetHashCode() } /// - /// Compare a string span with another using source identity semantics - same source, same position, same length. + /// Compare a string span with another using source identity + /// semantics - same source, same position, same length. /// /// The other span. /// True if the spans are the same. public bool Equals(TextSpan other) { - return string.Equals(Source, other.Source) && Position.Absolute == other.Position.Absolute; + return ReferenceEquals(Source, other.Source) && + Position.Absolute == other.Position.Absolute && + Length == other.Length; } /// @@ -170,7 +173,7 @@ public TextSpan Until(TextSpan next) next.EnsureHasValue(); if (next.Source != Source) throw new ArgumentException("The spans are on different source strings.", nameof(next)); #endif - var charCount = Length - next.Length; + var charCount = next.Position.Absolute - Position.Absolute; return First(charCount); } diff --git a/src/Superpower/Model/TokenListParserResult`2.cs b/src/Superpower/Model/TokenListParserResult`2.cs index 69f382e..7e9eeaa 100644 --- a/src/Superpower/Model/TokenListParserResult`2.cs +++ b/src/Superpower/Model/TokenListParserResult`2.cs @@ -86,7 +86,7 @@ public T Value get { if (!HasValue) - throw new InvalidOperationException("TokenResult has no value."); + throw new InvalidOperationException($"{nameof(TokenListParserResult)} has no value."); return _value; } } diff --git a/src/Superpower/Parsers/Numerics.cs b/src/Superpower/Parsers/Numerics.cs index 57b18ea..cd511e7 100644 --- a/src/Superpower/Parsers/Numerics.cs +++ b/src/Superpower/Parsers/Numerics.cs @@ -35,7 +35,7 @@ public static class Numerics public static TextParser Natural { get; } = input => { var next = input.ConsumeChar(); - if (!next.HasValue || !char.IsDigit(next.Value)) + if (!next.HasValue || !CharInfo.IsLatinDigit(next.Value)) return Result.Empty(input, ExpectedDigit); TextSpan remainder; @@ -43,7 +43,7 @@ public static class Numerics { remainder = next.Remainder; next = remainder.ConsumeChar(); - } while (next.HasValue && char.IsDigit(next.Value)); + } while (next.HasValue && CharInfo.IsLatinDigit(next.Value)); return Result.Value(input.Until(remainder), input, remainder); }; @@ -55,7 +55,7 @@ public static class Numerics { var next = input.ConsumeChar(); - if (!next.HasValue || !char.IsDigit(next.Value)) + if (!next.HasValue || !CharInfo.IsLatinDigit(next.Value)) return Result.Empty(input, ExpectedDigit); TextSpan remainder; @@ -65,7 +65,7 @@ public static class Numerics val = 10 * val + (uint)(next.Value - '0'); remainder = next.Remainder; next = remainder.ConsumeChar(); - } while (next.HasValue && char.IsDigit(next.Value)); + } while (next.HasValue && CharInfo.IsLatinDigit(next.Value)); return Result.Value(val, input, remainder); }; @@ -77,7 +77,7 @@ public static class Numerics { var next = input.ConsumeChar(); - if (!next.HasValue || !char.IsDigit(next.Value)) + if (!next.HasValue || !CharInfo.IsLatinDigit(next.Value)) return Result.Empty(input, ExpectedDigit); TextSpan remainder; @@ -87,7 +87,7 @@ public static class Numerics val = 10 * val + (ulong)(next.Value - '0'); remainder = next.Remainder; next = remainder.ConsumeChar(); - } while (next.HasValue && char.IsDigit(next.Value)); + } while (next.HasValue && CharInfo.IsLatinDigit(next.Value)); return Result.Value(val, input, remainder); }; @@ -105,7 +105,7 @@ public static class Numerics if (next.Value == '-' || next.Value == '+') next = next.Remainder.ConsumeChar(); - if (!next.HasValue || !char.IsDigit(next.Value)) + if (!next.HasValue || !CharInfo.IsLatinDigit(next.Value)) return Result.Empty(input, ExpectedDigit); TextSpan remainder; @@ -113,7 +113,7 @@ public static class Numerics { remainder = next.Remainder; next = remainder.ConsumeChar(); - } while (next.HasValue && char.IsDigit(next.Value)); + } while (next.HasValue && CharInfo.IsLatinDigit(next.Value)); return Result.Value(input.Until(remainder), input, remainder); }; @@ -140,7 +140,7 @@ public static class Numerics next = next.Remainder.ConsumeChar(); } - if (!next.HasValue || !char.IsDigit(next.Value)) + if (!next.HasValue || !CharInfo.IsLatinDigit(next.Value)) return Result.Empty(input, ExpectedDigit); TextSpan remainder; @@ -150,7 +150,7 @@ public static class Numerics val = 10 * val + (next.Value - '0'); remainder = next.Remainder; next = remainder.ConsumeChar(); - } while (next.HasValue && char.IsDigit(next.Value)); + } while (next.HasValue && CharInfo.IsLatinDigit(next.Value)); if (negative) val = -val; @@ -180,7 +180,7 @@ public static class Numerics next = next.Remainder.ConsumeChar(); } - if (!next.HasValue || !char.IsDigit(next.Value)) + if (!next.HasValue || !CharInfo.IsLatinDigit(next.Value)) return Result.Empty(input, ExpectedDigit); TextSpan remainder; @@ -190,7 +190,7 @@ public static class Numerics val = 10 * val + (next.Value - '0'); remainder = next.Remainder; next = remainder.ConsumeChar(); - } while (next.HasValue && char.IsDigit(next.Value)); + } while (next.HasValue && CharInfo.IsLatinDigit(next.Value)); if (negative) val = -val; diff --git a/src/Superpower/Superpower.csproj b/src/Superpower/Superpower.csproj index d7abbc5..9a5eb03 100644 --- a/src/Superpower/Superpower.csproj +++ b/src/Superpower/Superpower.csproj @@ -1,9 +1,14 @@ - + + + netstandard1.0;netstandard2.0 + + + net45;netstandard1.0;netstandard2.0 + A parser combinator library for C# - 2.1.0 + 2.2.0 Datalust;Superpower Contributors;Sprache Contributors - net45;netstandard1.0 true true Superpower diff --git a/src/Superpower/Util/CharInfo.cs b/src/Superpower/Util/CharInfo.cs index cd17ebb..cb513a8 100644 --- a/src/Superpower/Util/CharInfo.cs +++ b/src/Superpower/Util/CharInfo.cs @@ -16,14 +16,19 @@ namespace Superpower.Util { static class CharInfo { + public static bool IsLatinDigit(char ch) + { + return ch >= '0' && ch <= '9'; + } + public static bool IsHexDigit(char ch) { - return char.IsDigit(ch) || ch >= 'a' && ch <= 'f' || ch >= 'A' && ch <= 'F'; + return IsLatinDigit(ch) || ch >= 'a' && ch <= 'f' || ch >= 'A' && ch <= 'F'; } public static int HexValue(char ch) { - if (char.IsDigit(ch)) + if (IsLatinDigit(ch)) return ch - '0'; if (ch >= 'a' && ch <= 'f') diff --git a/test/Superpower.Benchmarks/ArithmeticExpressionScenario/ArithmeticExpressionTokenizer.cs b/test/Superpower.Benchmarks/ArithmeticExpressionScenario/ArithmeticExpressionTokenizer.cs index e5c4b25..e97326e 100644 --- a/test/Superpower.Benchmarks/ArithmeticExpressionScenario/ArithmeticExpressionTokenizer.cs +++ b/test/Superpower.Benchmarks/ArithmeticExpressionScenario/ArithmeticExpressionTokenizer.cs @@ -26,13 +26,14 @@ protected override IEnumerable> Tokenize(TextS { ArithmeticExpressionToken charToken; - if (char.IsDigit(next.Value)) + var ch = next.Value; + if (ch >= '0' && ch <= '9') { var integer = Numerics.Integer(next.Location); next = integer.Remainder.ConsumeChar(); yield return Result.Value(ArithmeticExpressionToken.Number, integer.Location, integer.Remainder); } - else if (_operators.TryGetValue(next.Value, out charToken)) + else if (_operators.TryGetValue(ch, out charToken)) { yield return Result.Value(charToken, next.Location, next.Remainder); next = next.Remainder.ConsumeChar(); diff --git a/test/Superpower.Benchmarks/NumberListScenario/NumberListTokenizer.cs b/test/Superpower.Benchmarks/NumberListScenario/NumberListTokenizer.cs index d2713d2..bf6be9e 100644 --- a/test/Superpower.Benchmarks/NumberListScenario/NumberListTokenizer.cs +++ b/test/Superpower.Benchmarks/NumberListScenario/NumberListTokenizer.cs @@ -15,7 +15,8 @@ protected override IEnumerable> Tokenize(TextSpan span) do { - if (char.IsDigit(next.Value)) + var ch = next.Value; + if (ch >= '0' && ch <= '9') { var integer = Numerics.Integer(next.Location); next = integer.Remainder.ConsumeChar(); diff --git a/test/Superpower.Tests/ArithmeticExpressionScenario/ArithmeticExpressionTokenizer.cs b/test/Superpower.Tests/ArithmeticExpressionScenario/ArithmeticExpressionTokenizer.cs index 730fce0..b7cce09 100644 --- a/test/Superpower.Tests/ArithmeticExpressionScenario/ArithmeticExpressionTokenizer.cs +++ b/test/Superpower.Tests/ArithmeticExpressionScenario/ArithmeticExpressionTokenizer.cs @@ -24,13 +24,14 @@ protected override IEnumerable> Tokenize(TextS do { - if (char.IsDigit(next.Value)) + var ch = next.Value; + if (ch >= '0' && ch <= '9') { var natural = Numerics.Natural(next.Location); next = natural.Remainder.ConsumeChar(); yield return Result.Value(ArithmeticExpressionToken.Number, natural.Location, natural.Remainder); } - else if (_operators.TryGetValue(next.Value, out var charToken)) + else if (_operators.TryGetValue(ch, out var charToken)) { yield return Result.Value(charToken, next.Location, next.Remainder); next = next.Remainder.ConsumeChar(); diff --git a/test/Superpower.Tests/Combinators/ManyCombinatorTests.cs b/test/Superpower.Tests/Combinators/ManyCombinatorTests.cs index c91bef9..0581809 100644 --- a/test/Superpower.Tests/Combinators/ManyCombinatorTests.cs +++ b/test/Superpower.Tests/Combinators/ManyCombinatorTests.cs @@ -57,5 +57,30 @@ public void TokenManyFailsWithPartialItemMatch() var list = ab.Many(); AssertParser.Fails(list, "ababa"); } + + [Fact] + public void ManySucceedsWithBacktrackedPartialItemMatch() + { + var ab = Character.EqualTo('a').Then(_ => Character.EqualTo('b')); + var ac = Character.EqualTo('a').Then(_ => Character.EqualTo('c')); + var list = Span.MatchedBy(ab.Try().Many().Then(_ => ac)); + AssertParser.SucceedsWithAll(list, "ababac"); + } + + [Fact] + public void ManyReportsCorrectErrorPositionForNonBacktrackingPartialItemMatch() + { + var ab = Character.EqualTo('a').Then(_ => Character.EqualTo('b')); + var list = ab.Many(); + AssertParser.FailsWithMessage(list, "ababac", "Syntax error (line 1, column 6): unexpected `c`, expected `b`."); + } + + [Fact] + public void ManyReportsCorrectRemainderForBacktrackingPartialItemMatch() + { + var ab = Character.EqualTo('a').Then(_ => Character.EqualTo('b')); + var list = Span.MatchedBy(ab.Try().Many()).Select(s => s.ToStringValue()); + AssertParser.SucceedsWith(list, "ababac", "abab"); + } } } diff --git a/test/Superpower.Tests/Display/PresentationTests.cs b/test/Superpower.Tests/Display/PresentationTests.cs index d1b82ca..7f21990 100644 --- a/test/Superpower.Tests/Display/PresentationTests.cs +++ b/test/Superpower.Tests/Display/PresentationTests.cs @@ -1,6 +1,7 @@ using Superpower.Display; using Superpower.Tests.SExpressionScenario; using Xunit; +using Superpower.Parsers; namespace Superpower.Tests.Display { @@ -19,5 +20,18 @@ public void DescriptionAttributeIsInterrogatedForDisplay() var display = Presentation.FormatExpectation(SExpressionToken.LParen); Assert.Equal("open parenthesis", display); } + [Fact] + public void ProperNameIsDisplayedWhenNonGraphicalCausesFailure() + { + var result=Character.EqualTo('a').TryParse("\x2007"); + + Assert.Equal("Syntax error (line 1, column 1): unexpected U+2007 figure space, expected `a`.", result.ToString()); + } + [Fact] + public void ProperNameIsDisplayedWhenNonGraphicalIsFailed() + { + var result=Character.EqualTo('\x2007').TryParse("a"); + Assert.Equal("Syntax error (line 1, column 1): unexpected `a`, expected U+2007 figure space.", result.ToString()); + } } } diff --git a/test/Superpower.Tests/NumberListScenario/NumberListTokenizer.cs b/test/Superpower.Tests/NumberListScenario/NumberListTokenizer.cs index cbbae7c..8200455 100644 --- a/test/Superpower.Tests/NumberListScenario/NumberListTokenizer.cs +++ b/test/Superpower.Tests/NumberListScenario/NumberListTokenizer.cs @@ -22,7 +22,8 @@ protected override IEnumerable> Tokenize(TextSpan span) do { - if (char.IsDigit(next.Value)) + var ch = next.Value; + if (ch >= '0' && ch <= '9') { var integer = Numerics.Integer(next.Location); next = integer.Remainder.ConsumeChar(); diff --git a/test/Superpower.Tests/Parsers/NumericsTests.cs b/test/Superpower.Tests/Parsers/NumericsTests.cs index 1cb1604..e3474a6 100644 --- a/test/Superpower.Tests/Parsers/NumericsTests.cs +++ b/test/Superpower.Tests/Parsers/NumericsTests.cs @@ -15,6 +15,8 @@ public class NumericsTests [InlineData("1.1", false)] [InlineData("a", false)] [InlineData("", false)] + [InlineData("\u0669\u0661\u0660", false)] // 910 in Arabic + [InlineData("9\u0661\u0660", false)] // 9 in Latin then 10 in Arabic public void IntegersAreRecognized(string input, bool isMatch) { AssertParser.FitsTheory(Numerics.Integer, input, isMatch); @@ -29,6 +31,8 @@ public void IntegersAreRecognized(string input, bool isMatch) [InlineData("1.1", false)] [InlineData("a", false)] [InlineData("", false)] + [InlineData("\u0669\u0661\u0660", false)] // 910 in Arabic + [InlineData("9\u0661\u0660", false)] // 9 in Latin then 10 in Arabic public void NaturalNumbersAreRecognized(string input, bool isMatch) { AssertParser.FitsTheory(Numerics.Natural, input, isMatch); @@ -44,6 +48,8 @@ public void NaturalNumbersAreRecognized(string input, bool isMatch) [InlineData("0123456789abcdef", true)] [InlineData("g", false)] [InlineData("", false)] + [InlineData("\u0669\u0661\u0660", false)] // 910 in Arabic + [InlineData("9\u0661\u0660", false)] // 9 in Latin then 10 in Arabic public void HexDigitsAreRecognized(string input, bool isMatch) { AssertParser.FitsTheory(Numerics.HexDigits, input, isMatch); diff --git a/test/Superpower.Tests/SExpressionScenario/SExpressionTokenizer.cs b/test/Superpower.Tests/SExpressionScenario/SExpressionTokenizer.cs index 094ca9a..0e7c727 100644 --- a/test/Superpower.Tests/SExpressionScenario/SExpressionTokenizer.cs +++ b/test/Superpower.Tests/SExpressionScenario/SExpressionTokenizer.cs @@ -25,7 +25,7 @@ protected override IEnumerable> Tokenize(TextSpan span) yield return Result.Value(SExpressionToken.RParen, next.Location, next.Remainder); next = next.Remainder.ConsumeChar(); } - else if (char.IsDigit(next.Value)) + else if (next.Value >= '0' && next.Value <= '9') { var integer = Numerics.Integer(next.Location); next = integer.Remainder.ConsumeChar(); diff --git a/test/Superpower.Tests/StringSpanTests.cs b/test/Superpower.Tests/StringSpanTests.cs index fa63dde..f5e87a3 100644 --- a/test/Superpower.Tests/StringSpanTests.cs +++ b/test/Superpower.Tests/StringSpanTests.cs @@ -13,6 +13,52 @@ public void ADefaultSpanHasNoValue() Assert.Throws(() => span.ToStringValue()); } + [Fact] + public void IdenticalSpansAreEqual() + { + var source = "123"; + var t1 = new TextSpan(source, Position.Zero, 1); + var t2 = new TextSpan(source, Position.Zero, 1); + Assert.Equal(t1, t2); + } + + [Fact] + public void SpansFromDifferentSourcesAreNotEqual() + { + string source1 = "123", source2 = "1234".Substring(0, 3); + var t1 = new TextSpan(source1, Position.Zero, 1); + var t2 = new TextSpan(source2, Position.Zero, 1); + Assert.NotEqual(t1, t2); + } + + [Fact] + public void DifferentLengthSpansAreNotEqual() + { + var source = "123"; + var t1 = new TextSpan(source, Position.Zero, 1); + var t2 = new TextSpan(source, Position.Zero, 2); + Assert.NotEqual(t1, t2); + } + + [Fact] + public void EqualSpansAreEqualCase65() + { + var source = "123"; + var one = Position.Zero.Advance(source[0]); + var t1 = new TextSpan(source); + var t2 = new TextSpan(source, one, 1); + Assert.Equal("1", t1.Until(t2).ToStringValue()); + } + + [Fact] + public void SpansAtDifferentPositionsAreNotEqual() + { + var source = "111"; + var t1 = new TextSpan(source, Position.Zero, 1); + var t2 = new TextSpan(source, new Position(1, 1, 1), 1); + Assert.NotEqual(t1, t2); + } + [Theory] [InlineData("Hello", 0, 5, "Hello")] [InlineData("Hello", 1, 4, "ello")]