Permalink
Browse files

re2c ready

  • Loading branch information...
1 parent 357e9b3 commit 9254c24fdf181065dd29d2d084dc72343a6cdd7a @tokuhirom committed Apr 26, 2012
Showing with 445 additions and 220 deletions.
  1. +1 −0 Build.PL
  2. +1 −220 lib/Nana/Parser.xs
  3. +222 −0 lib/Nana/toke.c
  4. +220 −0 lib/Nana/toke.re
  5. +1 −0 lib/Nana/toke.re.PL
View
@@ -45,6 +45,7 @@ my $build = $class->new(
},
PL_files => {
'Config.tra.PL' => 'share/lib/Config.tra',
+ 'lib/Nana/toke.re.PL' => 'lib/Nana/toke.c',
'lib/Nana/Token.pm.PL' => ['lib/Nana/Token.pm', 'lib/Nana/token.h'],
},
use_tap_harness => 1,
View
@@ -55,226 +55,7 @@ int is_opening(char c) {
) ? 1 : 0;
}
-/**
- * Take a token from string.
- *
- * @args src: source string.
- * @args len: length for 'src'.
- * @args output *used: used chars in src
- * @args output *found_end: make true if found __END__
- * @args output *lineno_inc: incremented line numbers.
- * @return int token id.
- */
-int token_op(char *src, size_t len, int *used, int *found_end, int *lineno_inc) {
-#define HAVE2(c) (len-*used>=2)
-#define HAVE3(c) (len-*used>=3)
-#define ALPHA2(c) (HAVE2() && isALPHA(*(p+1)))
-#define IDENT3(c) (HAVE3() && (isALNUM(*(p+2)) || *(p+2)=='_'))
-#define CHAR2(c) (HAVE2() && *(p+1) == (c))
-#define CHAR3(c) (HAVE3() && *(p+2) == (c))
-#define SIMPLEOP(type,_used) do { *used+=_used; return type; } while (0)
- *used = skip_ws(src, len, found_end, lineno_inc);
- if (*found_end) {
- return TOKEN_EOF;
- }
- if (*used == len) {
- return TOKEN_EOF;
- }
- char *p = src+*used;
-
- switch (*p) {
- case '/':
- if (CHAR2('=')) {
- SIMPLEOP(TOKEN_DIV_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_DIV, 1);
- }
- case '%':
- if (CHAR2('=')) {
- SIMPLEOP(TOKEN_MOD_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_MOD, 1);
- }
- case ',':
- SIMPLEOP(TOKEN_COMMA, 1);
- case '!':
- if (CHAR2('=')) {
- SIMPLEOP(TOKEN_NOT_EQUAL, 2);
- } else if (CHAR2('~')) {
- SIMPLEOP(TOKEN_REGEXP_NOT_MATCH, 2);
- } else {
- SIMPLEOP(TOKEN_NOT, 1);
- }
- case '=':
- if (CHAR2('=')) {
- SIMPLEOP(TOKEN_EQUAL_EQUAL, 2);
- } else if (CHAR2('>')) { /* => */
- SIMPLEOP(TOKEN_FAT_COMMA, 2);
- } else if (CHAR2('~')) { /* =~ */
- SIMPLEOP(TOKEN_REGEXP_MATCH, 2);
- } else {
- SIMPLEOP(TOKEN_ASSIGN, 1);
- }
- case '^':
- if (CHAR2('=')) {
- SIMPLEOP(TOKEN_XOR_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_XOR, 1);
- }
- case '.':
- if (CHAR2('.')) {
- if (CHAR3('.')) {
- SIMPLEOP(TOKEN_DOTDOTDOT, 3);
- } else {
- SIMPLEOP(TOKEN_DOTDOT, 2);
- }
- } else {
- SIMPLEOP(TOKEN_DOT, 1);
- }
- break;
- case '|':
- if (CHAR2('|')) {
- if (CHAR3('=')) {
- SIMPLEOP(TOKEN_OROR_ASSIGN, 3);
- } else {
- SIMPLEOP(TOKEN_OROR, 2);
- }
- } else if (CHAR2('=')) {
- SIMPLEOP(TOKEN_OR_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_OR, 1);
- }
- case '&':
- if (CHAR2('&')) {
- SIMPLEOP(TOKEN_ANDAND, 2);
- } else if (CHAR2('=')) {
- SIMPLEOP(TOKEN_AND_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_AND, 1);
- }
- break;
- case '<':
- if (CHAR2('<')) {
- if (CHAR3('=')) {
- SIMPLEOP(TOKEN_LSHIFT_ASSIGN, 3);
- } else if (CHAR3('\'')) { /* <<' */
- SIMPLEOP(TOKEN_HEREDOC_SQ_START, 3);
- } else {
- SIMPLEOP(TOKEN_LSHIFT, 2);
- }
- } else if (CHAR2('=')) {
- if (CHAR3('>')) { /* <=> */
- SIMPLEOP(TOKEN_CMP, 3);
- } else {
- SIMPLEOP(TOKEN_GE, 2);
- }
- } else {
- SIMPLEOP(TOKEN_GT, 1);
- }
- break;
- case '>':
- if (CHAR2('>')) {
- if (CHAR3('=')) {
- SIMPLEOP(TOKEN_RSHIFT_ASSIGN, 3);
- } else {
- SIMPLEOP(TOKEN_RSHIFT, 2);
- }
- } else if (CHAR2('=')) {
- SIMPLEOP(TOKEN_LE, 2);
- } else {
- SIMPLEOP(TOKEN_LT, 1);
- }
- break;
- case '\\':
- SIMPLEOP(TOKEN_REF, 1);
- break;
- case '~':
- SIMPLEOP(TOKEN_TILDE, 1);
- break;
- case '$':
- if (CHAR2('{')) { /* ${ */
- SIMPLEOP(TOKEN_DEREF, 2);
- }
- break;
- case '*':
- if (CHAR2('*')) {
- if (CHAR3('=')) {
- SIMPLEOP(TOKEN_POW_ASSIGN, 3);
- } else {
- SIMPLEOP(TOKEN_POW, 2);
- }
- } else if (CHAR2('=')) { /* *= */
- SIMPLEOP(TOKEN_MUL_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_MUL, 1);
- }
- break;
- case '+':
- if (CHAR2('+')) {
- SIMPLEOP(TOKEN_PLUSPLUS, 2);
- } else if (CHAR2('=')) { /* += */
- SIMPLEOP(TOKEN_PLUS_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_PLUS, 1);
- }
- break;
- case '{':
- SIMPLEOP(TOKEN_LBRACE, 1);
- break;
- case '(':
- SIMPLEOP(TOKEN_LPAREN, 1);
- break;
- case 'b':
- if (CHAR2('\'')) {
- SIMPLEOP(TOKEN_BYTES_SQ, 2);
- } else if (CHAR2('"')) {
- SIMPLEOP(TOKEN_BYTES_DQ, 2);
- }
- break;
- case 'q':
- if (CHAR2('q') && HAVE3() && is_opening(*(p+2))) { /* qq{ */
- SIMPLEOP(TOKEN_STRING_QQ_START, 3);
- }
- if (CHAR2('r') && HAVE3() && is_opening(*(p+2))) { /* qr{ */
- SIMPLEOP(TOKEN_REGEXP_QR_START, 3);
- }
- if (CHAR2('w') && HAVE3() && is_opening(*(p+2))) { /* qw{ */
- SIMPLEOP(TOKEN_QW_START, 3);
- }
- if (HAVE2() && is_opening(*(p+1))) { /* q{ */
- SIMPLEOP(TOKEN_STRING_Q_START, 2);
- }
- break;
- case '"':
- SIMPLEOP(TOKEN_STRING_DQ, 1);
- break;
- case '\'':
- SIMPLEOP(TOKEN_STRING_SQ, 1);
- break;
- case '[':
- SIMPLEOP(TOKEN_LBRACKET, 1);
- break;
- case '-':
- /* [[qr{^-(?![=a-z>-])}, '-'], [qr{^\+(?![\+=])}, '+']]) */
- if (ALPHA2() && !IDENT3()) {
- SIMPLEOP(TOKEN_FILETEST, 2);
- }
-
- if (CHAR2('-')) { /* -- */
- SIMPLEOP(TOKEN_MINUSMINUS, 2);
- } else if (CHAR2('>')) { /* -> */
- SIMPLEOP(TOKEN_LAMBDA, 2);
- } else if (CHAR2('=')) { /* -= */
- SIMPLEOP(TOKEN_MINUS_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_MINUS, 1);
- }
- break;
- }
- return TOKEN_EOF;
-#undef CHAR2
-#undef SIMPLEOP
-}
+#include "toke.c"
MODULE = Nana::Parser PACKAGE = Nana::Parser
Oops, something went wrong. Retry.

0 comments on commit 9254c24

Please sign in to comment.