Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files

Add escape character to lexer

  • Loading branch information...
vierbergenlars committed Aug 8, 2013
1 parent ec2af3d commit ecf8077a051c721ec9e50b903e899485dd32a44c
Showing with 46 additions and 19 deletions.
  1. +6 −1 lib/vierbergenlars/Norch/QueryParser/Lexer.php
  2. +40 −18 test/queryparser/lexer.php
@@ -23,6 +23,9 @@ public static function tokenize($string)
while($i < $len) {
$c = $string[$i];
switch($c) {
case '\\': // Escape character
$current_token->addData($string[++$i]);
break;
case ' ':
self::push($tokens, $current_token, $i);
break;
@@ -85,7 +88,9 @@ static private function push(&$tokens, &$current_token, $i)
static private function readEncString(Token $current_token, $string, &$i)
{
while(++$i < strlen($string)) {
if($string[$i] != '"') {
if($string[$i] == '\\') {
$current_token->addData($string[++$i]);
} else if($string[$i] != '"') {
$current_token->addData($string[$i]);
} else {
break;
@@ -51,15 +51,13 @@ function testLexerTokenize()
}
$this->assertTrue($ex);
$tokens = L::tokenize('blabal "search query" "field": "long value"');
$this->assertEqual($tokens[0]->getType(), Token::T_STRING);
$this->assertEqual($tokens[0]->getData(), 'blabal');
$this->assertEqual($tokens[1]->getType(), Token::T_STRING);
$this->assertEqual($tokens[1]->getData(), 'search query');
$this->assertEqual($tokens[2]->getType(), Token::T_STRING);
$this->assertEqual($tokens[2]->getData(), 'field');
$this->assertEqual($tokens[3]->getType(), Token::T_FIELD_VALUE);
$this->assertEqual($tokens[3]->getData(), 'long value');
$ex = false;
try {
L::tokenize('blabal "search query" "field": "long value"');
} catch(ParseException $e) {
$ex = true;
}
$this->assertTrue($ex);
$tokens = L::tokenize('blabal "search query" "field:" "long value"');
$this->assertEqual($tokens[0]->getType(), Token::T_STRING);
@@ -87,16 +85,40 @@ function testLexerTokenize()
$this->assertEqual($tokens[0]->getData(), 'field');
$this->assertEqual($tokens[1]->getType(), Token::T_FIELD_WEIGHT);
$this->assertEqual($tokens[1]->getData(), '25');
$this->assertEqual($tokens[2]->getType(), Token::T_FIELD_VALUE);
$this->assertEqual($tokens[2]->getData(), 'long');
$this->assertEqual($tokens[2]->getType(), Token::T_FIELD_NAME);
$this->assertEqual($tokens[2]->getData(), 'field');
$this->assertEqual($tokens[3]->getType(), Token::T_FIELD_VALUE);
$this->assertEqual($tokens[3]->getData(), 'long');
$this->assertEqual($tokens[4]->getType(), Token::T_STRING);
$this->assertEqual($tokens[4]->getData(), 'value');
$this->assertEqual($tokens[5]->getType(), Token::T_FIELD_NAME);
$this->assertEqual($tokens[5]->getData(), 'field2');
$this->assertEqual($tokens[6]->getType(), Token::T_FIELD_WEIGHT);
$this->assertEqual($tokens[6]->getData(), '-1');
$this->assertEqual($tokens[7]->getType(), Token::T_STRING);
$this->assertEqual($tokens[7]->getData(), 'sfi');
$tokens = L::tokenize('title^2 @body "bla bla"');
$this->assertEqual($tokens[0]->getType(), Token::T_FIELD_NAME);
$this->assertEqual($tokens[0]->getData(), 'title');
$this->assertEqual($tokens[1]->getType(), Token::T_FIELD_WEIGHT);
$this->assertEqual($tokens[1]->getData(), '2');
$this->assertEqual($tokens[2]->getType(), Token::T_FIELD_SEARCH);
$this->assertEqual($tokens[2]->getData(), 'body');
$this->assertEqual($tokens[3]->getType(), Token::T_STRING);
$this->assertEqual($tokens[3]->getData(), 'bla bla');
$tokens = L::tokenize('title\^2 \@body "bla bla\" bla" title\: zz');
$this->assertEqual($tokens[0]->getType(), Token::T_STRING);
$this->assertEqual($tokens[0]->getData(), 'title^2');
$this->assertEqual($tokens[1]->getType(), Token::T_STRING);
$this->assertEqual($tokens[1]->getData(), '@body');
$this->assertEqual($tokens[2]->getType(), Token::T_STRING);
$this->assertEqual($tokens[2]->getData(), 'bla bla" bla');
$this->assertEqual($tokens[3]->getType(), Token::T_STRING);
$this->assertEqual($tokens[3]->getData(), 'value');
$this->assertEqual($tokens[4]->getType(), Token::T_FIELD_NAME);
$this->assertEqual($tokens[4]->getData(), 'field2');
$this->assertEqual($tokens[5]->getType(), Token::T_FIELD_WEIGHT);
$this->assertEqual($tokens[5]->getData(), '-1');
$this->assertEqual($tokens[6]->getType(), Token::T_STRING);
$this->assertEqual($tokens[6]->getData(), 'sfi');
$this->assertEqual($tokens[3]->getData(), 'title:');
$this->assertEqual($tokens[4]->getType(), Token::T_STRING);
$this->assertEqual($tokens[4]->getData(), 'zz');
}
}

0 comments on commit ecf8077

Please sign in to comment.
You can’t perform that action at this time.