update tokenizer extension

php · Feb 11, 2015 · 8f862e7 · 8f862e7
1 parent d9d6f0c
commit 8f862e7
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 3 deletions.
diff --git a/ext/tokenizer/tests/semi_reserved_001.phpt b/ext/tokenizer/tests/semi_reserved_001.phpt
@@ -0,0 +1,55 @@
+--TEST--
+Test token_get_all() function with semi-reserved grammar
+--FILE--
+<?php
+
+$tokens = token_get_all('<?php
+class {
+    const CONST = "const";
+    public $public = "public";
+    protected function function(){}
+}
+
+const CONST = "const"; // syntax error
+
+function function(){}  // syntax error
+
+');
+
+// var_dump($tokens);
+
+echo token_name($tokens[5][0]), PHP_EOL;
+echo token_name($tokens[7][0]), PHP_EOL;
+echo PHP_EOL;
+echo token_name($tokens[14][0]), PHP_EOL;
+echo token_name($tokens[16][0]), PHP_EOL;
+echo PHP_EOL;
+echo token_name($tokens[25][0]), PHP_EOL;
+echo token_name($tokens[27][0]), PHP_EOL;
+echo PHP_EOL;
+echo token_name($tokens[35][0]), PHP_EOL;
+echo token_name($tokens[37][0]), PHP_EOL;
+echo PHP_EOL;
+echo token_name($tokens[46][0]), PHP_EOL;
+echo token_name($tokens[48][0]), PHP_EOL;
+
+echo "\nDone\n";
+
+--EXPECT--
+
+T_CONST
+T_STRING
+
+T_PUBLIC
+T_VARIABLE
+
+T_FUNCTION
+T_STRING
+
+T_CONST
+T_CONST
+
+T_FUNCTION
+T_FUNCTION
+
+Done
diff --git a/ext/tokenizer/tokenizer_data.c b/ext/tokenizer/tokenizer_data.c
@@ -39,6 +39,7 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_LOGICAL_AND", T_LOGICAL_AND, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_PRINT", T_PRINT, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_YIELD", T_YIELD, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_POW_EQUAL", T_POW_EQUAL, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_SR_EQUAL", T_SR_EQUAL, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_SL_EQUAL", T_SL_EQUAL, CONST_CS | CONST_PERSISTENT);
@@ -84,7 +85,6 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_FINAL", T_FINAL, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ABSTRACT", T_ABSTRACT, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_STATIC", T_STATIC, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_LNUMBER", T_LNUMBER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DNUMBER", T_DNUMBER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_STRING", T_STRING, CONST_CS | CONST_PERSISTENT);
@@ -177,6 +177,7 @@ char *get_token_type_name(int token_type)
 		case T_LOGICAL_AND: return "T_LOGICAL_AND";
 		case T_PRINT: return "T_PRINT";
 		case T_YIELD: return "T_YIELD";
+		case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW";
 		case T_POW_EQUAL: return "T_POW_EQUAL";
 		case T_SR_EQUAL: return "T_SR_EQUAL";
 		case T_SL_EQUAL: return "T_SL_EQUAL";
@@ -222,7 +223,6 @@ char *get_token_type_name(int token_type)
 		case T_FINAL: return "T_FINAL";
 		case T_ABSTRACT: return "T_ABSTRACT";
 		case T_STATIC: return "T_STATIC";
-		case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW";
 		case T_LNUMBER: return "T_LNUMBER";
 		case T_DNUMBER: return "T_DNUMBER";
 		case T_STRING: return "T_STRING";

diff --git a/ext/tokenizer/tokenizer_data_gen.sh b/ext/tokenizer/tokenizer_data_gen.sh
@@ -16,7 +16,7 @@ echo '/*
    +----------------------------------------------------------------------+
    | PHP Version 7                                                        |
    +----------------------------------------------------------------------+
-   | Copyright (c) 1997-2014 The PHP Group                                |
+   | Copyright (c) 1997-2015 The PHP Group                                |
    +----------------------------------------------------------------------+
    | This source file is subject to version 3.01 of the PHP license,      |
    | that is bundled with this package in the file LICENSE, and is        |