Revert "Handle scanner error in first place (don't hide them from ext/tokenizer) and cheaper whitespace handlig."

This reverts commit 0d6da03.

Author: dstogov

Zend/zend_compile.c

@@ -1690,16 +1690,25 @@ int zendlex(zend_parser_stack_elem *elem) /* {{{ */
 	ZVAL_UNDEF(&zv);
 	start_lineno = CG(zend_lineno);
 	retval = lex_scan(&zv);
+	if (EG(exception)) {
+		return T_ERROR;
+	}
 
-	if (retval >= T_WHITESPACE) {
-		if (EXPECTED(retval < T_OPEN_TAG_WITH_ECHO)) {
+	switch (retval) {
+		case T_COMMENT:
+		case T_DOC_COMMENT:
+		case T_OPEN_TAG:
+		case T_WHITESPACE:
 			goto again;
-		} else if (retval == T_OPEN_TAG_WITH_ECHO) {
-			retval = T_ECHO;
-		} else if (retval == T_CLOSE_TAG) {
+
+		case T_CLOSE_TAG:
 			retval = ';'; /* implicit ; */
-		}
-	} else if (Z_TYPE(zv) != IS_UNDEF) {
+			break;
+		case T_OPEN_TAG_WITH_ECHO:
+			retval = T_ECHO;
+			break;
+	}
+	if (Z_TYPE(zv) != IS_UNDEF) {
 		elem->ast = zend_ast_create_zval_with_lineno(&zv, 0, start_lineno);
 	}
 

Zend/zend_language_parser.y

@@ -203,6 +203,12 @@ static YYSIZE_T zend_yytnamerr(char*, const char*);
 %token T_TRAIT_C         "__TRAIT__ (T_TRAIT_C)"
 %token T_METHOD_C        "__METHOD__ (T_METHOD_C)"
 %token T_FUNC_C          "__FUNCTION__ (T_FUNC_C)"
+%token T_COMMENT         "comment (T_COMMENT)"
+%token T_DOC_COMMENT     "doc comment (T_DOC_COMMENT)"
+%token T_OPEN_TAG        "open tag (T_OPEN_TAG)"
+%token T_OPEN_TAG_WITH_ECHO "open tag with echo (T_OPEN_TAG_WITH_ECHO)"
+%token T_CLOSE_TAG       "close tag (T_CLOSE_TAG)"
+%token T_WHITESPACE      "whitespace (T_WHITESPACE)"
 %token T_START_HEREDOC   "heredoc start (T_START_HEREDOC)"
 %token T_END_HEREDOC     "heredoc end (T_END_HEREDOC)"
 %token T_DOLLAR_OPEN_CURLY_BRACES "${ (T_DOLLAR_OPEN_CURLY_BRACES)"
@@ -216,14 +222,6 @@ static YYSIZE_T zend_yytnamerr(char*, const char*);
 %token T_POW             "** (T_POW)"
 %token T_POW_EQUAL       "**= (T_POW_EQUAL)"
 
-/* Special tokens (ignorred by PHP compiler, T_WHITESPACE must be first) */
-%token T_WHITESPACE      "whitespace (T_WHITESPACE)"
-%token T_COMMENT         "comment (T_COMMENT)"
-%token T_DOC_COMMENT     "doc comment (T_DOC_COMMENT)"
-%token T_OPEN_TAG        "open tag (T_OPEN_TAG)"
-%token T_OPEN_TAG_WITH_ECHO "open tag with echo (T_OPEN_TAG_WITH_ECHO)"
-%token T_CLOSE_TAG       "close tag (T_CLOSE_TAG)"
-
 /* Token used to force a parse error from the lexer */
 %token T_ERROR