-
Notifications
You must be signed in to change notification settings - Fork 0
/
LinedLexer.cs
36 lines (29 loc) · 951 Bytes
/
LinedLexer.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
using System.Collections.Generic;
using System.IO;
namespace Lexepars
{
public class LinedLexer : LexerBase<TextReader>
{
public LinedLexer(params MatchableTokenKindSpec[] kinds)
: base(null, kinds)
{ }
public override IEnumerable<Token> Tokenize(TextReader textReader)
{
var text = new LinedInputText(textReader);
while (text.ReadLine())
while (!text.EndOfLine)
{
var current = GetToken(text);
if (current == null)
{
yield return CreateUnknownToken(text);
yield break;
}
text.Advance(current.Lexeme.Length);
if (SkippableTokenKinds.Contains(current.Kind))
continue;
yield return current;
}
}
}
}