flesh out get all tokens method

boyter · Jul 24, 2020 · a597848 · a597848
1 parent bd05e7f
commit a597848
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 1 deletion.
diff --git a/processor/parser/lexer.go b/processor/parser/lexer.go
@@ -76,6 +76,7 @@ func (l *Lexer) Next() byte {
 	return 0
 }
 
+// Return the next token for the input
 func (l *Lexer) NextToken() Token {
 
 	// at the end so return end token
@@ -156,5 +157,14 @@ func (l *Lexer) NextToken() Token {
 }
 
 func (l *Lexer) Tokens() []Token {
-	return nil
+	var tokens []Token
+
+	t := l.NextToken()
+
+	for t.Type != "END" {
+		tokens = append(tokens, t)
+		t = l.NextToken()
+	}
+
+	return tokens
 }
diff --git a/processor/parser/lexer_test.go b/processor/parser/lexer_test.go
@@ -380,3 +380,13 @@ func TestNextTokenMultiple(t *testing.T) {
 		t.Error(`expected PAREN_CLOSE got`, token.Type)
 	}
 }
+
+func TestTokens(t *testing.T) {
+	lex := NewLexer(`(something AND else) OR (other NOT this)`)
+
+	tokens := lex.Tokens()
+
+	if len(tokens) != 11 {
+		t.Error("expected 11 tokens got", len(tokens))
+	}
+}