-
Notifications
You must be signed in to change notification settings - Fork 96
/
SExpressionTokenizer.cs
55 lines (49 loc) · 2 KB
/
SExpressionTokenizer.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
using Superpower;
using Superpower.Parsers;
using Superpower.Model;
using System.Collections.Generic;
namespace Superpower.Tests.SExpressionScenario
{
class SExpressionTokenizer : Tokenizer<SExpressionToken>
{
protected override IEnumerable<Result<SExpressionToken>> Tokenize(TextSpan span)
{
var next = SkipWhiteSpace(span);
if (!next.HasValue)
yield break;
do
{
if (next.Value == '(')
{
yield return Result.Value(SExpressionToken.LParen, next.Location, next.Remainder);
next = next.Remainder.ConsumeChar();
}
else if (next.Value == ')')
{
yield return Result.Value(SExpressionToken.RParen, next.Location, next.Remainder);
next = next.Remainder.ConsumeChar();
}
else if (next.Value >= '0' && next.Value <= '9')
{
var integer = Numerics.Integer(next.Location);
next = integer.Remainder.ConsumeChar();
yield return Result.Value(SExpressionToken.Number, integer.Location, integer.Remainder);
if (next.HasValue && !char.IsPunctuation(next.Value) && !char.IsWhiteSpace(next.Value))
{
yield return Result.Empty<SExpressionToken>(next.Location, new[] {"whitespace", "punctuation"});
}
}
else
{
var beginIdentifier = next.Location;
while (next.HasValue && char.IsLetterOrDigit(next.Value))
{
next = next.Remainder.ConsumeChar();
}
yield return Result.Value(SExpressionToken.Atom, beginIdentifier, next.Location);
}
next = SkipWhiteSpace(next.Location);
} while (next.HasValue);
}
}
}