Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Use a TokenStream object instead of embedding the tokens in the lexer

  • Loading branch information...
commit 4f76cbb5ea80be14502207f86dff8b4e1e4549c3 1 parent 156e43a
@relaxnow authored
View
20 src/QueryLang/v3/Lexer.php
@@ -6,7 +6,7 @@ class Lexer
{
private $_input;
private $_pos = 0;
- private $_tokens;
+ private $_tokens = array();
/**
* @param string $input
@@ -16,23 +16,6 @@ public function __construct($input)
$this->_input = $input;
}
- /**
- * @param int $pos
- * @return Token
- */
- public function lookAhead($pos = 0)
- {
- return $this->_tokens[$pos];
- }
-
- /**
- * @return Token
- */
- public function nextToken()
- {
- return array_shift($this->_tokens);
- }
-
public function lex()
{
while (!empty($this->_input)) {
@@ -48,6 +31,7 @@ public function lex()
);
}
$this->_tokens[] = new Token('EOS', '', $this->_pos);
+ return new TokenStream($this->_tokens);
}
protected function _match($type, $regex, $skip = false)
View
51 src/QueryLang/v3/Parser.php
@@ -12,34 +12,31 @@
*/
class Parser
{
- protected $_lexer;
+ protected $_tokenStream;
- public function __construct($string)
+ public function __construct($input)
{
- $this->_lexer = new Lexer($string);
+ $lexer = new Lexer($input);
+ $this->_tokenStream = $lexer->lex();
}
public function parse()
{
- $this->_lexer->lex();
+ $query = $this->_query();
- return $this->_query();
+ $this->_tokenStream->expect('EOS');
+ return $query;
}
protected function _query()
{
- return $this->_orQuery();
- }
-
- protected function _orQuery()
- {
$query = new Node\Query('OR');
$leftTerm = $this->_andQuery();
$query->add($leftTerm);
- while ($this->_predict()->getType() === 'OR') {
- $this->_accept('OR');
+ while ($this->_tokenStream->look()->getType() === 'OR') {
+ $this->_tokenStream->expect('OR');
$rightTerm = $this->_andQuery();
$query->add($rightTerm);
}
@@ -54,8 +51,8 @@ protected function _andQuery()
$leftTerm = $this->_term();
$query->add($leftTerm);
- while ($this->_predict()->getType() === 'AND') {
- $this->_accept('AND');
+ while ($this->_tokenStream->look()->getType() === 'AND') {
+ $this->_tokenStream->expect('AND');
$rightTerm = $this->_term();
$query->add($rightTerm);
}
@@ -65,17 +62,17 @@ protected function _andQuery()
protected function _term()
{
- $nextTokenType = $this->_predict()->getType();
+ $nextTokenType = $this->_tokenStream->look()->getType();
if ($nextTokenType === 'LeftParen') {
- $this->_accept('LeftParen');
+ $this->_tokenStream->expect('LeftParen');
$query = $this->_query();
- $this->_accept('RightParen');
+ $this->_tokenStream->expect('RightParen');
return $query;
}
else if ($nextTokenType === 'TermValue') {
- $termValue = $this->_accept('TermValue');
- return new Node\Term($termValue);
+ $termValue = $this->_tokenStream->expect('TermValue');
+ return new Node\Term($termValue->getValue());
}
else {
throw new SyntaxException(
@@ -83,20 +80,4 @@ protected function _term()
);
}
}
-
- protected function _predict()
- {
- return $this->_lexer->lookAhead();
- }
-
- protected function _accept($tokenType)
- {
- $token = $this->_lexer->nextToken();
- if ($token->getType() !== $tokenType) {
- throw new SyntaxException(
- "Unexpected token " . $token->getType() . " at " . $token->getPos() . ", expecting: $tokenType"
- );
- }
- return $token->getValue();
- }
}
View
54 src/QueryLang/v3/TokenStream.php
@@ -0,0 +1,54 @@
+<?php
+
+namespace QueryLang\v3;
+
+class TokenStream
+{
+ private $_pos = 0;
+ private $_tokens = array();
+
+ public function __construct($tokens)
+ {
+ $this->_tokens = $tokens;
+ }
+
+ /**
+ * @param int $pos
+ * @return Token
+ */
+ public function look($pos = 1)
+ {
+ return $this->_tokens[$this->_pos + --$pos];
+ }
+
+ /**
+ * @param $tokenType
+ * @return Token
+ * @throws SyntaxException
+ */
+ public function expect($tokenType)
+ {
+ /** @var $token Token */
+ $token = $this->_tokens[$this->_pos];
+ if ($token->getType() !== $tokenType) {
+ $excerpt = "";
+ if (isset($this->_tokens[$this->_pos - 1])) {
+ $excerpt .= $this->_tokens[$this->_pos - 1]->getValue();
+ }
+ $excerpt .= $token->getValue();
+ if (isset($this->_tokens[$this->_pos + 1])) {
+ $excerpt .= $this->_tokens[$this->_pos + 1]->getValue();
+ }
+ var_dump($this->_tokens);
+
+ throw new SyntaxException(
+ "Unexpected token " . $token->getType() .
+ " at " . $token->getPos() .
+ ' in "' . $excerpt . '"' .
+ ", expecting: $tokenType"
+ );
+ }
+ $this->_pos++;
+ return $token;
+ }
+}
Please sign in to comment.
Something went wrong with that request. Please try again.