From 8f862e7b88b013c5be04c41b26248e04ff466928 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?M=C3=A1rcio=20Almada?= <marcio3w@gmail.com>
Date: Wed, 11 Feb 2015 10:43:50 -0300
Subject: [PATCH] update tokenizer extension

---
 ext/tokenizer/tests/semi_reserved_001.phpt | 55 ++++++++++++++++++++++
 ext/tokenizer/tokenizer_data.c             |  4 +-
 ext/tokenizer/tokenizer_data_gen.sh        |  2 +-
 3 files changed, 58 insertions(+), 3 deletions(-)
 create mode 100644 ext/tokenizer/tests/semi_reserved_001.phpt

diff --git a/ext/tokenizer/tests/semi_reserved_001.phpt b/ext/tokenizer/tests/semi_reserved_001.phpt
new file mode 100644
index 0000000000000..60f95806ec217
--- /dev/null
+++ b/ext/tokenizer/tests/semi_reserved_001.phpt
@@ -0,0 +1,55 @@
+--TEST--
+Test token_get_all() function with semi-reserved grammar
+--FILE--
+<?php
+
+$tokens = token_get_all('<?php
+class {
+    const CONST = "const";
+    public $public = "public";
+    protected function function(){}
+}
+
+const CONST = "const"; // syntax error
+
+function function(){}  // syntax error
+
+');
+
+// var_dump($tokens);
+
+echo token_name($tokens[5][0]), PHP_EOL;
+echo token_name($tokens[7][0]), PHP_EOL;
+echo PHP_EOL;
+echo token_name($tokens[14][0]), PHP_EOL;
+echo token_name($tokens[16][0]), PHP_EOL;
+echo PHP_EOL;
+echo token_name($tokens[25][0]), PHP_EOL;
+echo token_name($tokens[27][0]), PHP_EOL;
+echo PHP_EOL;
+echo token_name($tokens[35][0]), PHP_EOL;
+echo token_name($tokens[37][0]), PHP_EOL;
+echo PHP_EOL;
+echo token_name($tokens[46][0]), PHP_EOL;
+echo token_name($tokens[48][0]), PHP_EOL;
+
+echo "\nDone\n";
+
+--EXPECT--
+
+T_CONST
+T_STRING
+
+T_PUBLIC
+T_VARIABLE
+
+T_FUNCTION
+T_STRING
+
+T_CONST
+T_CONST
+
+T_FUNCTION
+T_FUNCTION
+
+Done
diff --git a/ext/tokenizer/tokenizer_data.c b/ext/tokenizer/tokenizer_data.c
index a707c639a9c60..b323a03e7f8f6 100644
--- a/ext/tokenizer/tokenizer_data.c
+++ b/ext/tokenizer/tokenizer_data.c
@@ -39,6 +39,7 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_LOGICAL_AND", T_LOGICAL_AND, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_PRINT", T_PRINT, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_YIELD", T_YIELD, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_POW_EQUAL", T_POW_EQUAL, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_SR_EQUAL", T_SR_EQUAL, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_SL_EQUAL", T_SL_EQUAL, CONST_CS | CONST_PERSISTENT);
@@ -84,7 +85,6 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_FINAL", T_FINAL, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ABSTRACT", T_ABSTRACT, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_STATIC", T_STATIC, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_LNUMBER", T_LNUMBER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DNUMBER", T_DNUMBER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_STRING", T_STRING, CONST_CS | CONST_PERSISTENT);
@@ -177,6 +177,7 @@ char *get_token_type_name(int token_type)
 		case T_LOGICAL_AND: return "T_LOGICAL_AND";
 		case T_PRINT: return "T_PRINT";
 		case T_YIELD: return "T_YIELD";
+		case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW";
 		case T_POW_EQUAL: return "T_POW_EQUAL";
 		case T_SR_EQUAL: return "T_SR_EQUAL";
 		case T_SL_EQUAL: return "T_SL_EQUAL";
@@ -222,7 +223,6 @@ char *get_token_type_name(int token_type)
 		case T_FINAL: return "T_FINAL";
 		case T_ABSTRACT: return "T_ABSTRACT";
 		case T_STATIC: return "T_STATIC";
-		case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW";
 		case T_LNUMBER: return "T_LNUMBER";
 		case T_DNUMBER: return "T_DNUMBER";
 		case T_STRING: return "T_STRING";
diff --git a/ext/tokenizer/tokenizer_data_gen.sh b/ext/tokenizer/tokenizer_data_gen.sh
index 0966367020e50..13384c849045a 100755
--- a/ext/tokenizer/tokenizer_data_gen.sh
+++ b/ext/tokenizer/tokenizer_data_gen.sh
@@ -16,7 +16,7 @@ echo '/*
    +----------------------------------------------------------------------+
    | PHP Version 7                                                        |
    +----------------------------------------------------------------------+
-   | Copyright (c) 1997-2014 The PHP Group                                |
+   | Copyright (c) 1997-2015 The PHP Group                                |
    +----------------------------------------------------------------------+
    | This source file is subject to version 3.01 of the PHP license,      |
    | that is bundled with this package in the file LICENSE, and is        |