Permalink
Browse files

- Rewrite scanner to be based on re2c instead of flex

  The full patch is available as:
  http://php.net/~helly/php-re2c-5.3-20080316.diff.txt
  This is against php-re2c repository version 98
  An older patch against version 97 is available under:
  http://php.net/~helly/php-re2c-97-20080316.diff.txt
  • Loading branch information...
1 parent eb8f83a commit af316021e8f69896cd0d246114962e48b973972f Marcus Boerger committed Mar 16, 2008
Showing with 1,049 additions and 2,995 deletions.
  1. +2 −2 Makefile.frag
  2. +1 −0 NEWS
  3. +0 −186 Zend/FlexLexer.h
  4. +3 −3 Zend/Makefile.am
  5. +0 −15 Zend/Zend.m4
  6. +0 −1 Zend/configure.in
  7. +0 −1,648 Zend/flex.skl
  8. +7 −0 Zend/tests/bug42767.phpt
  9. +14 −21 Zend/zend.c
  10. +12 −51 Zend/zend_compile.c
  11. +0 −8 Zend/zend_compile.h
  12. +29 −53 Zend/zend_globals.h
  13. +6 −5 Zend/zend_globals_macros.h
  14. +2 −20 Zend/zend_highlight.c
  15. +1 −1 Zend/zend_ini_parser.y
  16. +0 −1 Zend/zend_ini_scanner.h
  17. +98 −119 Zend/zend_ini_scanner.l
  18. +9 −19 Zend/zend_language_scanner.h
  19. +235 −505 Zend/zend_language_scanner.l
  20. +6 −5 Zend/zend_stack.c
  21. +242 −68 Zend/zend_stream.c
  22. +40 −23 Zend/zend_stream.h
  23. +1 −1 Zend/zend_strtod.c
  24. +1 −1 Zend/zend_vm_def.h
  25. +4 −4 Zend/zend_vm_execute.h
  26. +4 −67 acinclude.m4
  27. +29 −2 configure.in
  28. +1 −1 ext/spl/php_spl.c
  29. +4 −4 ext/standard/tests/strings/006.phpt
  30. +3 −3 ext/standard/tests/strings/007.phpt
  31. +4 −13 ext/standard/tests/strings/highlight_file.phpt
  32. +1 −1 ext/standard/tests/strings/strip_tags_basic1.phpt
  33. +1 −1 ext/standard/tests/strings/strip_tags_basic2.phpt
  34. +1 −1 ext/standard/tests/strings/strip_tags_error.phpt
  35. +1 −1 ext/standard/tests/strings/strip_tags_variation1.phpt
  36. +1 −1 ext/standard/tests/strings/strip_tags_variation10.phpt
  37. +1 −1 ext/standard/tests/strings/strip_tags_variation2.phpt
  38. +1 −1 ext/standard/tests/strings/strip_tags_variation3.phpt
  39. +1 −1 ext/standard/tests/strings/strip_tags_variation4.phpt
  40. +1 −1 ext/standard/tests/strings/strip_tags_variation5.phpt
  41. +1 −1 ext/standard/tests/strings/strip_tags_variation6.phpt
  42. +1 −1 ext/standard/tests/strings/strip_tags_variation7.phpt
  43. +1 −1 ext/standard/tests/strings/strip_tags_variation8.phpt
  44. +1 −1 ext/standard/tests/strings/strip_tags_variation9.phpt
  45. +2 −2 ext/tokenizer/Makefile.frag
  46. +2 −11 ext/tokenizer/tests/token_get_all_variation14.phpt
  47. +2 −60 ext/tokenizer/tokenizer.c
  48. +35 −13 main/main.c
  49. +0 −8 sapi/apache/mod_php5.c
  50. +0 −8 sapi/apache_hooks/mod_php5.c
  51. +3 −3 sapi/cgi/cgi_main.c
  52. +2 −2 sapi/cgi/tests/007.phpt
  53. +4 −1 sapi/cli/php_cli.c
  54. +11 −12 sapi/cli/tests/010.phpt
  55. +105 −0 sapi/cli/tests/016.phpt
  56. +106 −0 sapi/cli/tests/017.phpt
  57. +3 −3 sapi/tests/test003.phpt
  58. +3 −3 sapi/tests/test004.phpt
  59. +0 −6 win32/build/config.w32
View
@@ -7,7 +7,7 @@ $(builddir)/zend_language_scanner.lo: $(srcdir)/zend_language_parser.h
$(builddir)/zend_ini_scanner.lo: $(srcdir)/zend_ini_parser.h
$(srcdir)/zend_language_scanner.c: $(srcdir)/zend_language_scanner.l
- @$(LEX) -Pzend -S$(srcdir)/flex.skl -o$@ -i $(srcdir)/zend_language_scanner.l
+ $(RE2C) $(RE2C_FLAGS) --case-inverted -cbdFt $(srcdir)/zend_language_scanner_defs.h -o$@ $(srcdir)/zend_language_scanner.l
$(srcdir)/zend_language_parser.h: $(srcdir)/zend_language_parser.c
$(srcdir)/zend_language_parser.c: $(srcdir)/zend_language_parser.y
@@ -18,6 +18,6 @@ $(srcdir)/zend_ini_parser.c: $(srcdir)/zend_ini_parser.y
@$(YACC) -p ini_ -v -d $(srcdir)/zend_ini_parser.y -o $@
$(srcdir)/zend_ini_scanner.c: $(srcdir)/zend_ini_scanner.l
- @$(LEX) -Pini_ -S$(srcdir)/flex.skl -o$@ -i $(srcdir)/zend_ini_scanner.l
+ $(RE2C) $(RE2C_FLAGS) --case-inverted -cbdFt $(srcdir)/zend_ini_scanner_defs.h -o$@ $(srcdir)/zend_ini_scanner.l
$(builddir)/zend_indent.lo $(builddir)/zend_highlight.lo $(builddir)/zend_compile.lo: $(srcdir)/zend_language_parser.h
View
1 NEWS
@@ -79,6 +79,7 @@ PHP NEWS
- Added PREG_BAD_UTF8_OFFSET_ERROR constant. (Nuno)
- Improved PHP runtime speed and memory usage:
+ . Replaced flex based scanner with re2c based scanner. (Marcus, Nuno, Scott)
. Added garbage collector. (David Wang, Dmitry).
. Improved PHP binary size and startup speed with GCC4 visibility control.
(Nuno)
View
@@ -1,186 +0,0 @@
-// $Header$
-
-// FlexLexer.h -- define interfaces for lexical analyzer classes generated
-// by flex
-
-// Copyright (c) 1993 The Regents of the University of California.
-// All rights reserved.
-//
-// This code is derived from software contributed to Berkeley by
-// Kent Williams and Tom Epperly.
-//
-// Redistribution and use in source and binary forms with or without
-// modification are permitted provided that: (1) source distributions retain
-// this entire copyright notice and comment, and (2) distributions including
-// binaries display the following acknowledgement: ``This product includes
-// software developed by the University of California, Berkeley and its
-// contributors'' in the documentation or other materials provided with the
-// distribution and in all advertising materials mentioning features or use
-// of this software. Neither the name of the University nor the names of
-// its contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-
-// THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
-// WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
-// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
-
-// This file defines FlexLexer, an abstract class which specifies the
-// external interface provided to flex C++ lexer objects, and yyFlexLexer,
-// which defines a particular lexer class.
-//
-// If you want to create multiple lexer classes, you use the -P flag
-// to rename each yyFlexLexer to some other xxFlexLexer. You then
-// include <FlexLexer.h> in your other sources once per lexer class:
-//
-// #undef yyFlexLexer
-// #define yyFlexLexer xxFlexLexer
-// #include <FlexLexer.h>
-//
-// #undef yyFlexLexer
-// #define yyFlexLexer zzFlexLexer
-// #include <FlexLexer.h>
-// ...
-
-#ifndef FLEXLEXER_H
-// Never included before - need to define base class.
-#define FLEXLEXER_H
-#include <iostream.h>
-
-extern "C++" {
-
-struct yy_buffer_state;
-typedef int yy_state_type;
-
-class FlexLexer {
-public:
- virtual ~FlexLexer() { }
-
- const char* YYText() { return yytext; }
- int YYLeng() { return yyleng; }
-
- virtual void
- yy_switch_to_buffer( struct yy_buffer_state* new_buffer ) = 0;
- virtual struct yy_buffer_state*
- yy_create_buffer( istream* s, int size ) = 0;
- virtual void yy_delete_buffer( struct yy_buffer_state* b ) = 0;
- virtual void yyrestart( istream* s ) = 0;
-
- virtual int yylex() = 0;
-
- // Call yylex with new input/output sources.
- int yylex( istream* new_in, ostream* new_out = 0 )
- {
- switch_streams( new_in, new_out );
- return yylex();
- }
-
- // Switch to new input/output streams. A nil stream pointer
- // indicates "keep the current one".
- virtual void switch_streams( istream* new_in = 0,
- ostream* new_out = 0 ) = 0;
-
- int lineno() const { return yylineno; }
-
- int debug() const { return yy_flex_debug; }
- void set_debug( int flag ) { yy_flex_debug = flag; }
-
-protected:
- char* yytext;
- int yyleng;
- int yylineno; // only maintained if you use %option yylineno
- int yy_flex_debug; // only has effect with -d or "%option debug"
-};
-
-}
-#endif
-
-#if defined(yyFlexLexer) || ! defined(yyFlexLexerOnce)
-// Either this is the first time through (yyFlexLexerOnce not defined),
-// or this is a repeated include to define a different flavor of
-// yyFlexLexer, as discussed in the flex man page.
-#define yyFlexLexerOnce
-
-class yyFlexLexer : public FlexLexer {
-public:
- // arg_yyin and arg_yyout default to the cin and cout, but we
- // only make that assignment when initializing in yylex().
- yyFlexLexer( istream* arg_yyin = 0, ostream* arg_yyout = 0 );
-
- virtual ~yyFlexLexer();
-
- void yy_switch_to_buffer( struct yy_buffer_state* new_buffer );
- struct yy_buffer_state* yy_create_buffer( istream* s, int size );
- void yy_delete_buffer( struct yy_buffer_state* b );
- void yyrestart( istream* s );
-
- virtual int yylex();
- virtual void switch_streams( istream* new_in, ostream* new_out );
-
-protected:
- virtual int LexerInput( char* buf, int max_size );
- virtual void LexerOutput( const char* buf, int size );
- virtual void LexerError( const char* msg );
-
- void yyunput( int c, char* buf_ptr );
- int yyinput();
-
- void yy_load_buffer_state();
- void yy_init_buffer( struct yy_buffer_state* b, istream* s );
- void yy_flush_buffer( struct yy_buffer_state* b );
-
- int yy_start_stack_ptr;
- int yy_start_stack_depth;
- int* yy_start_stack;
-
- void yy_push_state( int new_state );
- void yy_pop_state();
- int yy_top_state();
-
- yy_state_type yy_get_previous_state();
- yy_state_type yy_try_NUL_trans( yy_state_type current_state );
- int yy_get_next_buffer();
-
- istream* yyin; // input source for default LexerInput
- ostream* yyout; // output sink for default LexerOutput
-
- struct yy_buffer_state* yy_current_buffer;
-
- // yy_hold_char holds the character lost when yytext is formed.
- char yy_hold_char;
-
- // Number of characters read into yy_ch_buf.
- int yy_n_chars;
-
- // Points to current character in buffer.
- char* yy_c_buf_p;
-
- int yy_init; // whether we need to initialize
- int yy_start; // start state number
-
- // Flag which is used to allow yywrap()'s to do buffer switches
- // instead of setting up a fresh yyin. A bit of a hack ...
- int yy_did_buffer_switch_on_eof;
-
- // The following are not always needed, but may be depending
- // on use of certain flex features (like REJECT or yymore()).
-
- yy_state_type yy_last_accepting_state;
- char* yy_last_accepting_cpos;
-
- yy_state_type* yy_state_buf;
- yy_state_type* yy_state_ptr;
-
- char* yy_full_match;
- int* yy_full_state;
- int yy_full_lp;
-
- int yy_lp;
- int yy_looking_for_trail_begin;
-
- int yy_more_flag;
- int yy_more_len;
- int yy_more_offset;
- int yy_prev_more_offset;
-};
-
-#endif
View
@@ -17,7 +17,7 @@ libZend_la_SOURCES=\
zend_objects_API.c zend_ts_hash.c zend_stream.c \
zend_default_classes.c \
zend_iterators.c zend_interfaces.c zend_exceptions.c \
- zend_strtod.c zend_multibyte.c
+ zend_strtod.c
libZend_la_LDFLAGS =
libZend_la_LIBADD = @ZEND_EXTRA_LIBS@
@@ -31,7 +31,7 @@ zend_ini_scanner.lo: zend_ini_parser.h
# Language parser/scanner rules
zend_language_scanner.c: $(srcdir)/zend_language_scanner.l
- $(LEX) -Pzend -S$(srcdir)/flex.skl -o$@ -i $(srcdir)/zend_language_scanner.l
+ $(RE2C) $(RE2C_FLAGS) --case-inverted -cbdFt $(srcdir)/zend_language_scanner_defs.h -o$@ $(srcdir)/zend_language_scanner.l
zend_language_parser.h: zend_language_parser.c
zend_language_parser.c: $(srcdir)/zend_language_parser.y
@@ -43,7 +43,7 @@ zend_ini_parser.c: $(srcdir)/zend_ini_parser.y
$(YACC) -p ini_ -v -d $(srcdir)/zend_ini_parser.y -o zend_ini_parser.c
zend_ini_scanner.c: $(srcdir)/zend_ini_scanner.l
- $(LEX) -Pini_ -S$(srcdir)/flex.skl -o$@ -i $(srcdir)/zend_ini_scanner.l
+ $(RE2C) $(RE2C_FLAGS) --case-inverted -cbdFt $(srcdir)/zend_ini_scanner_defs.h -o$@ $(srcdir)/zend_ini_scanner.l
zend_ini_parser.h: zend_ini_parser.c
View
@@ -31,7 +31,6 @@ AC_DEFUN([LIBZEND_BASIC_CHECKS],[
AC_REQUIRE([AC_PROG_YACC])
AC_REQUIRE([AC_PROG_CC])
AC_REQUIRE([AC_PROG_CC_C_O])
-AC_REQUIRE([AC_PROG_LEX])
AC_REQUIRE([AC_HEADER_STDC])
LIBZEND_BISON_CHECK
@@ -155,13 +154,6 @@ AC_ARG_ENABLE(inline-optimization,
ZEND_INLINE_OPTIMIZATION=yes
])
-AC_ARG_ENABLE(zend-multibyte,
-[ --enable-zend-multibyte Compile with zend multibyte support], [
- ZEND_MULTIBYTE=$enableval
-],[
- ZEND_MULTIBYTE=no
-])
-
AC_MSG_CHECKING([virtual machine dispatch method])
AC_MSG_RESULT($PHP_ZEND_VM)
@@ -174,9 +166,6 @@ AC_MSG_RESULT($ZEND_INLINE_OPTIMIZATION)
AC_MSG_CHECKING(whether to enable Zend debugging)
AC_MSG_RESULT($ZEND_DEBUG)
-AC_MSG_CHECKING(whether to enable Zend multibyte)
-AC_MSG_RESULT($ZEND_MULTIBYTE)
-
case $PHP_ZEND_VM in
SWITCH)
AC_DEFINE(ZEND_VM_KIND,ZEND_VM_KIND_SWITCH,[virtual machine dispatch method])
@@ -211,10 +200,6 @@ if test "$ZEND_MAINTAINER_ZTS" = "yes"; then
LIBZEND_CPLUSPLUS_CHECKS
fi
-if test "$ZEND_MULTIBYTE" = "yes"; then
- AC_DEFINE(ZEND_MULTIBYTE, 1, [ ])
-fi
-
changequote({,})
if test -n "$GCC" && test "$ZEND_INLINE_OPTIMIZATION" != "yes"; then
INLINE_CFLAGS=`echo $ac_n "$CFLAGS $ac_c" | sed s/-O[0-9s]*//`
View
@@ -7,7 +7,6 @@ AM_CONFIG_HEADER(zend_config.h)
AM_SANITY_CHECK
AM_MAINTAINER_MODE
AC_PROG_CC
-AM_PROG_LEX
AM_PROG_CC_STDC
ZEND_VERSION=$VERSION
AC_ZEND_C_BIGENDIAN
Oops, something went wrong.

0 comments on commit af31602

Please sign in to comment.