/
LexerTests.cs
60 lines (52 loc) · 1.78 KB
/
LexerTests.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
using System.Collections.Generic;
using Should;
using Xunit;
namespace Parsley
{
public class LexerTests
{
private readonly TokenKind lower;
private readonly TokenKind upper;
private readonly TokenKind space;
public LexerTests()
{
lower = new Pattern("Lowercase", @"[a-z]+");
upper = new Pattern("Uppercase", @"[A-Z]+");
space = new Pattern("Space", @"\s", skippable: true);
}
private IEnumerable<Token> Tokenize(string input)
{
return new Lexer(lower, upper, space).Tokenize(input);
}
[Fact]
public void ProvidesEmptyEnumerableForEmptyText()
{
Tokenize("").ShouldBeEmpty();
}
[Fact]
public void UsesPrioritizedTokenMatchersToTokenize()
{
Tokenize("ABCdefGHI")
.ShouldList(t => t.ShouldEqual(upper, "ABC", 1, 1),
t => t.ShouldEqual(lower, "def", 1, 4),
t => t.ShouldEqual(upper, "GHI", 1, 7));
}
[Fact]
public void ProvidesTokenAtUnrecognizedInput()
{
Tokenize("ABC!def")
.ShouldList(t => t.ShouldEqual(upper, "ABC", 1, 1),
t => t.ShouldEqual(TokenKind.Unknown, "!def", 1, 4));
}
[Fact]
public void SkipsPastSkippableTokens()
{
Tokenize(" ").ShouldBeEmpty();
Tokenize(" ABC def GHI jkl ")
.ShouldList(t => t.ShouldEqual(upper, "ABC", 1, 2),
t => t.ShouldEqual(lower, "def", 1, 7),
t => t.ShouldEqual(upper, "GHI", 1, 13),
t => t.ShouldEqual(lower, "jkl", 1, 20));
}
}
}