Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Comparing changes

Choose two branches to see what's changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
base fork: tokuhirom/Nana
base: 8b3dfdd90a
...
head fork: tokuhirom/Nana
compare: 1d88e00ec5
Checking mergeability… Don't worry, you can still create the pull request.
  • 6 commits
  • 9 files changed
  • 0 commit comments
  • 1 contributor
View
3  Build.PL
@@ -7,7 +7,7 @@ my $class = Module::Build->subclass(
class => 'My::Builder',
code => q"
sub ACTION_test {
- exec(q{PERL5LIB=./util/:$PERL5LIB prove --source T --source TRA --source EGTRA --source TCC -r t/ t/*/*/*.tcc t/*/*/*.egtra t/tra/*.tra t/tra/*/*.tra});
+ exec($^X . q{ util/myprove --source T --source TRA --source EGTRA --source TCC -r t/ t/*/*/*.tcc t/*/*/*.egtra t/tra/*.tra t/tra/*/*.tra});
}
"
);
@@ -45,6 +45,7 @@ my $build = $class->new(
},
PL_files => {
'Config.tra.PL' => 'share/lib/Config.tra',
+ 'lib/Nana/toke.re.PL' => 'lib/Nana/toke.c',
'lib/Nana/Token.pm.PL' => ['lib/Nana/Token.pm', 'lib/Nana/token.h'],
},
use_tap_harness => 1,
View
233 lib/Nana/Parser.pm
@@ -971,6 +971,52 @@ rule('primary', [
return _regexp(substr($c, $used), q{/});
} elsif ($token_id == TOKEN_REGEXP_QR_START) { # qr{
return _regexp(substr($c, $used), _closechar(substr($c, $used-1, 1)));
+ } elsif ($token_id == TOKEN_QW_START) { # qw{
+ return _qw_literal(substr($c, $used), _closechar(substr($c, $used-1, 1)));
+ } elsif ($token_id ==TOKEN_HEREDOC_SQ_START) { # <<'
+ $c = substr($c, $used);
+ $c =~ s/^([^, \t\n']+)//
+ or die "Parsing failed on heredoc LINE $LINENO";
+ my $marker = $1;
+ ($c) = match($c, q{'})
+ or die "Parsing failed on heredoc LINE $LINENO";
+ my $buf = '';
+ push @HEREDOC_BUFS, \$buf;
+ push @HEREDOC_MARKERS, $marker;
+ return ($c, _node2('HEREDOC', $START, \$buf));
+ } elsif ($token_id ==TOKEN_BYTES_SQ) { # b'
+ return _bytes_sq(substr($c, $used), 0);
+ } elsif ($token_id ==TOKEN_BYTES_DQ) { # b"
+ return _bytes_dq(substr($c, $used), 0);
+ } elsif ($token_id == TOKEN_LPAREN) { # (
+ $c = substr($c, $used);
+ ($c, my $body) = expression($c)
+ or return;
+ ($c) = match($c, ")")
+ or return;
+ return ($c, _node2('()', $START, $body));
+ } elsif ($token_id == TOKEN_LBRACE) { # {
+ $c = substr($c, $used);
+ my @content;
+ while (my ($c2, $lhs) = assign_expression($c)) {
+ $lhs->[0] = 'IDENT' if $lhs->[0] eq 'PRIMARY_IDENT';
+ push @content, $lhs;
+ $c = $c2;
+ ($c2) = match($c, '=>')
+ or return;
+ # or die "Missing => in hash creation '@{[ substr($c, 10) ]}...' line $LINENO\n";
+ $c = $c2;
+ ($c, my $rhs) = assign_expression($c);
+ push @content, $rhs;
+
+ my ($c3) = match($c, ',');
+ last unless defined $c3;
+ $c = $c3;
+ }
+ ($c) = match($c, "}")
+ or return;
+ # or die "} not found on hash at line $LINENO";
+ return ($c, _node2('{}', $START, \@content));
} else {
return;
}
@@ -988,13 +1034,6 @@ rule('primary', [
or return;
return ($c, _node('INT', $1));
},
- \&bytes,
- sub {
- my $c = shift;
- ($c, my $ret) = _qw_literal($c)
- or return;
- ($c, $ret);
- },
sub {
my $c = shift;
$c =~ s/^my\b//
@@ -1055,86 +1094,29 @@ rule('primary', [
},
sub {
my $c = shift;
- ($c) = match($c, "{")
- or return;
- my @content;
- while (my ($c2, $lhs) = assign_expression($c)) {
- $lhs->[0] = 'IDENT' if $lhs->[0] eq 'PRIMARY_IDENT';
- push @content, $lhs;
- $c = $c2;
- ($c2) = match($c, '=>')
- or return;
- # or die "Missing => in hash creation '@{[ substr($c, 10) ]}...' line $LINENO\n";
- $c = $c2;
- ($c, my $rhs) = assign_expression($c);
- push @content, $rhs;
-
- my ($c3) = match($c, ',');
- last unless defined $c3;
- $c = $c3;
- }
- ($c) = match($c, "}")
- or return;
- # or die "} not found on hash at line $LINENO";
- return ($c, _node2('{}', $START, \@content));
- },
- sub {
- my $c = shift;
- ($c) = match($c, "(")
- or return;
- ($c, my $body) = expression($c);
- ($c) = match($c, ")")
- or return;
- return ($c, _node2('()', $START, $body));
- },
- sub {
- my $c = shift;
($c, my $word) = match($c, 'undef', 'true', 'false', 'self', '__FILE__', '__LINE__')
or return;
return ($c, _node2(uc($word), $START));
},
- sub {
- my $c = shift;
- ($c) = match($c, q{<<'})
- or return;
- $c =~ s/^([^, \t\n']+)//
- or die "Parsing failed on heredoc LINE $LINENO";
- my $marker = $1;
- ($c) = match($c, q{'})
- or die "Parsing failed on heredoc LINE $LINENO";
- my $buf = '';
- push @HEREDOC_BUFS, \$buf;
- push @HEREDOC_MARKERS, $marker;
- return ($c, _node2('HEREDOC', $START, \$buf));
- },
]);
-rule('_qw_literal', [
- sub {
- my $src = shift;
+sub _qw_literal {
+ my ($src, $close) = @_;
+ $close = quotemeta($close);
- $src =~ s!^qw([\(\[\!\{"])!!smx or return;
- my $close = quotemeta +{
- '(' => ')',
- '[' => ']',
- '{' => '}',
- '!' => '!',
- '"' => '"',
- }->{$1};
- my $ret = [];
- while (1) {
- ($src, my $got_end) = skip_ws($src);
- return if $got_end;
- if ($src =~ s!^([^ \t\Q$close\E]+)!!) {
- push @$ret, $1;
- } elsif ($src =~ s!^$close!!smx) {
- return ($src, _node('QW', $ret));
- } else {
- die "Parse failed in qw() literal: $src";
- }
+ my $ret = [];
+ while (1) {
+ ($src, my $got_end) = skip_ws($src);
+ return if $got_end;
+ if ($src =~ s!^([^ \t\Q$close\E]+)!!) {
+ push @$ret, $1;
+ } elsif ($src =~ s!^$close!!smx) {
+ return ($src, _node('QW', $ret));
+ } else {
+ die "Parse failed in qw() literal: $src";
}
}
-]);
+}
sub _regexp {
my ($src, $close) = @_;
@@ -1163,60 +1145,55 @@ sub _regexp {
return ($src, _node('REGEXP', $buf, $flags));
}
-rule('bytes', [
- sub {
- # TODO: escape chars, etc.
- my $src = shift;
+sub _bytes_dq {
+ # TODO: escape chars, etc.
+ my $src = shift;
- ($src) = match($src, q{b"})
- or return;
- my $buf = '';
- while (1) {
- if ($src =~ s/^"//) {
- last;
- } elsif (length($src) == 0) {
- die "Unexpected EOF in bytes literal line $START";
- } elsif ($src =~ s/^\\"//) {
- $buf .= q{"};
- } elsif ($src =~ s/^(\\[0-7]{3})//) {
- $buf .= $1;
- } elsif ($src =~ s/^(\\x[0-9a-f]{2})//) {
- $buf .= $1;
- } elsif ($src =~ s/^(.)//) {
- $buf .= $1;
- } else {
- die 'should not reach here';
- }
+ my $buf = '';
+ while (1) {
+ if ($src =~ s/^"//) {
+ last;
+ } elsif (length($src) == 0) {
+ die "Unexpected EOF in bytes literal line $START";
+ } elsif ($src =~ s/^\\"//) {
+ $buf .= q{"};
+ } elsif ($src =~ s/^(\\[0-7]{3})//) {
+ $buf .= $1;
+ } elsif ($src =~ s/^(\\x[0-9a-f]{2})//) {
+ $buf .= $1;
+ } elsif ($src =~ s/^(.)//) {
+ $buf .= $1;
+ } else {
+ die 'should not reach here';
}
- return ($src, _node('BYTES', $buf));
- },
- sub {
- # TODO: escape chars, etc.
- my $src = shift;
+ }
+ return ($src, _node('BYTES', $buf));
+}
- ($src) = match($src, q{b'})
- or return;
- my $buf = '';
- while (1) {
- if ($src =~ s/^'//) {
- last;
- } elsif (length($src) == 0) {
- die "Unexpected EOF in bytes literal line $START";
- } elsif ($src =~ s/^\\'//) {
- $buf .= q{'};
- } elsif ($src =~ s/^(\\[0-7]{3})//) {
- $buf .= $1;
- } elsif ($src =~ s/^(\\x[0-9a-f]{2})//) {
- $buf .= $1;
- } elsif ($src =~ s/^(.)//) {
- $buf .= $1;
- } else {
- die 'should not reach here';
- }
+sub _bytes_sq {
+ # TODO: escape chars, etc.
+ my $src = shift;
+
+ my $buf = '';
+ while (1) {
+ if ($src =~ s/^'//) {
+ last;
+ } elsif (length($src) == 0) {
+ die "Unexpected EOF in bytes literal line $START";
+ } elsif ($src =~ s/^\\'//) {
+ $buf .= q{'};
+ } elsif ($src =~ s/^(\\[0-7]{3})//) {
+ $buf .= $1;
+ } elsif ($src =~ s/^(\\x[0-9a-f]{2})//) {
+ $buf .= $1;
+ } elsif ($src =~ s/^(.)//) {
+ $buf .= $1;
+ } else {
+ die 'should not reach here';
}
- return ($src, _node('BYTES', $buf));
- },
-]);
+ }
+ return ($src, _node('BYTES', $buf));
+}
sub _dq_string {
# TODO: escape chars, etc.
View
203 lib/Nana/Parser.xs
@@ -55,208 +55,7 @@ int is_opening(char c) {
) ? 1 : 0;
}
-/**
- * Take a token from string.
- *
- * @args src: source string.
- * @args len: length for 'src'.
- * @args output *used: used chars in src
- * @args output *found_end: make true if found __END__
- * @args output *lineno_inc: incremented line numbers.
- * @return int token id.
- */
-int token_op(char *src, size_t len, int *used, int *found_end, int *lineno_inc) {
-#define HAVE2(c) (len-*used>=2)
-#define HAVE3(c) (len-*used>=3)
-#define ALPHA2(c) (HAVE2() && isALPHA(*(p+1)))
-#define IDENT3(c) (HAVE3() && (isALNUM(*(p+2)) || *(p+2)=='_'))
-#define CHAR2(c) (HAVE2() && *(p+1) == (c))
-#define CHAR3(c) (HAVE3() && *(p+2) == (c))
-#define SIMPLEOP(type,_used) do { *used+=_used; return type; } while (0)
- *used = skip_ws(src, len, found_end, lineno_inc);
- if (*found_end) {
- return TOKEN_EOF;
- }
- if (*used == len) {
- return TOKEN_EOF;
- }
- char *p = src+*used;
-
- switch (*p) {
- case '/':
- if (CHAR2('=')) {
- SIMPLEOP(TOKEN_DIV_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_DIV, 1);
- }
- case '%':
- if (CHAR2('=')) {
- SIMPLEOP(TOKEN_MOD_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_MOD, 1);
- }
- case ',':
- SIMPLEOP(TOKEN_COMMA, 1);
- case '!':
- if (CHAR2('=')) {
- SIMPLEOP(TOKEN_NOT_EQUAL, 2);
- } else if (CHAR2('~')) {
- SIMPLEOP(TOKEN_REGEXP_NOT_MATCH, 2);
- } else {
- SIMPLEOP(TOKEN_NOT, 1);
- }
- case '=':
- if (CHAR2('=')) {
- SIMPLEOP(TOKEN_EQUAL_EQUAL, 2);
- } else if (CHAR2('>')) { /* => */
- SIMPLEOP(TOKEN_FAT_COMMA, 2);
- } else if (CHAR2('~')) { /* =~ */
- SIMPLEOP(TOKEN_REGEXP_MATCH, 2);
- } else {
- SIMPLEOP(TOKEN_ASSIGN, 1);
- }
- case '^':
- if (CHAR2('=')) {
- SIMPLEOP(TOKEN_XOR_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_XOR, 1);
- }
- case '.':
- if (CHAR2('.')) {
- if (CHAR3('.')) {
- SIMPLEOP(TOKEN_DOTDOTDOT, 3);
- } else {
- SIMPLEOP(TOKEN_DOTDOT, 2);
- }
- } else {
- SIMPLEOP(TOKEN_DOT, 1);
- }
- break;
- case '|':
- if (CHAR2('|')) {
- if (CHAR3('=')) {
- SIMPLEOP(TOKEN_OROR_ASSIGN, 3);
- } else {
- SIMPLEOP(TOKEN_OROR, 2);
- }
- } else if (CHAR2('=')) {
- SIMPLEOP(TOKEN_OR_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_OR, 1);
- }
- case '&':
- if (CHAR2('&')) {
- SIMPLEOP(TOKEN_ANDAND, 2);
- } else if (CHAR2('=')) {
- SIMPLEOP(TOKEN_AND_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_AND, 1);
- }
- break;
- case '<':
- if (CHAR2('<')) {
- if (CHAR3('=')) {
- SIMPLEOP(TOKEN_LSHIFT_ASSIGN, 3);
- } else {
- SIMPLEOP(TOKEN_LSHIFT, 2);
- }
- } else if (CHAR2('=')) {
- if (CHAR3('>')) { /* <=> */
- SIMPLEOP(TOKEN_CMP, 3);
- } else {
- SIMPLEOP(TOKEN_GE, 2);
- }
- } else {
- SIMPLEOP(TOKEN_GT, 1);
- }
- break;
- case '>':
- if (CHAR2('>')) {
- if (CHAR3('=')) {
- SIMPLEOP(TOKEN_RSHIFT_ASSIGN, 3);
- } else {
- SIMPLEOP(TOKEN_RSHIFT, 2);
- }
- } else if (CHAR2('=')) {
- SIMPLEOP(TOKEN_LE, 2);
- } else {
- SIMPLEOP(TOKEN_LT, 1);
- }
- break;
- case '\\':
- SIMPLEOP(TOKEN_REF, 1);
- break;
- case '~':
- SIMPLEOP(TOKEN_TILDE, 1);
- break;
- case '$':
- if (CHAR2('{')) { /* ${ */
- SIMPLEOP(TOKEN_DEREF, 2);
- }
- break;
- case '*':
- if (CHAR2('*')) {
- if (CHAR3('=')) {
- SIMPLEOP(TOKEN_POW_ASSIGN, 3);
- } else {
- SIMPLEOP(TOKEN_POW, 2);
- }
- } else if (CHAR2('=')) { /* *= */
- SIMPLEOP(TOKEN_MUL_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_MUL, 1);
- }
- break;
- case '+':
- if (CHAR2('+')) {
- SIMPLEOP(TOKEN_PLUSPLUS, 2);
- } else if (CHAR2('=')) { /* += */
- SIMPLEOP(TOKEN_PLUS_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_PLUS, 1);
- }
- break;
- case 'q':
- if (CHAR2('q') && HAVE3() && is_opening(*(p+2))) { /* qq{ */
- SIMPLEOP(TOKEN_STRING_QQ_START, 3);
- }
- if (CHAR2('r') && HAVE3() && is_opening(*(p+2))) { /* qr{ */
- SIMPLEOP(TOKEN_REGEXP_QR_START, 3);
- }
- if (HAVE2() && is_opening(*(p+1))) { /* q{ */
- SIMPLEOP(TOKEN_STRING_Q_START, 2);
- }
- break;
- case '"':
- SIMPLEOP(TOKEN_STRING_DQ, 1);
- break;
- case '\'':
- SIMPLEOP(TOKEN_STRING_SQ, 1);
- break;
- case '[':
- SIMPLEOP(TOKEN_LBRACKET, 1);
- break;
- case '-':
- /* [[qr{^-(?![=a-z>-])}, '-'], [qr{^\+(?![\+=])}, '+']]) */
- if (ALPHA2() && !IDENT3()) {
- SIMPLEOP(TOKEN_FILETEST, 2);
- }
-
- if (CHAR2('-')) { /* -- */
- SIMPLEOP(TOKEN_MINUSMINUS, 2);
- } else if (CHAR2('>')) { /* -> */
- SIMPLEOP(TOKEN_LAMBDA, 2);
- } else if (CHAR2('=')) { /* -= */
- SIMPLEOP(TOKEN_MINUS_ASSIGN, 2);
- } else {
- SIMPLEOP(TOKEN_MINUS, 1);
- }
- break;
- }
- return TOKEN_EOF;
-#undef CHAR2
-#undef SIMPLEOP
-}
+#include "toke.c"
MODULE = Nana::Parser PACKAGE = Nana::Parser
View
6 lib/Nana/Token.pm.PL
@@ -57,6 +57,12 @@ my @tokens = grep { !$seen{$_}++ } qw(
TOKEN_STRING_DQ
TOKEN_STRING_SQ
TOKEN_REGEXP_QR_START
+ TOKEN_HEREDOC_SQ_START
+ TOKEN_BYTES_SQ
+ TOKEN_BYTES_DQ
+ TOKEN_LPAREN
+ TOKEN_LBRACE
+ TOKEN_QW_START
);
&output_header; &output_pm; exit 0;
View
572 lib/Nana/toke.c
@@ -0,0 +1,572 @@
+/* Generated by re2c 0.13.5 on Thu Apr 26 14:27:02 2012 */
+#line 1 "lib/Nana/toke.re"
+/* vim: set filetype=cpp: */
+#include<stdio.h>
+
+/**
+ * Take a token from string.
+ *
+ * @args src: source string.
+ * @args len: length for 'src'.
+ * @args output *used: used chars in src
+ * @args output *found_end: make true if found __END__
+ * @args output *lineno_inc: incremented line numbers.
+ * @return int token id.
+ */
+int token_op(char *src, size_t len, int *used, int *found_end, int *lineno_inc) {
+#define OP(type) do { *used+=(cursor-orig); return type; } while (0)
+ *used = skip_ws(src, len, found_end, lineno_inc);
+ if (*found_end) {
+ return TOKEN_EOF;
+ }
+ if (*used == len) {
+ return TOKEN_EOF;
+ }
+ char *cursor = src+*used;
+ char *orig = cursor;
+ char *marker = cursor;
+
+ #line 38 "lib/Nana/toke.re"
+
+
+
+#line 34 "lib/Nana/toke.c"
+ {
+ char yych;
+
+ yych = (char)*cursor;
+ switch (yych) {
+ case '!': goto yy16;
+ case '"': goto yy40;
+ case '$': goto yy34;
+ case '%': goto yy12;
+ case '&': goto yy26;
+ case '\'': goto yy42;
+ case '(': goto yy6;
+ case '*': goto yy35;
+ case '+': goto yy2;
+ case ',': goto yy14;
+ case '-': goto yy46;
+ case '.': goto yy22;
+ case '/': goto yy10;
+ case '<': goto yy8;
+ case '=': goto yy18;
+ case '>': goto yy28;
+ case '[': goto yy44;
+ case '\\': goto yy30;
+ case '^': goto yy20;
+ case 'b': goto yy4;
+ case 'q': goto yy39;
+ case '{': goto yy37;
+ case '|': goto yy24;
+ case '~': goto yy32;
+ default: goto yy48;
+ }
+yy2:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '+': goto yy129;
+ case '=': goto yy127;
+ default: goto yy3;
+ }
+yy3:
+#line 43 "lib/Nana/toke.re"
+ { OP(TOKEN_PLUS); }
+#line 76 "lib/Nana/toke.c"
+yy4:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '"': goto yy123;
+ case '\'': goto yy125;
+ default: goto yy5;
+ }
+yy5:
+#line 108 "lib/Nana/toke.re"
+ { return TOKEN_EOF; }
+#line 87 "lib/Nana/toke.c"
+yy6:
+ ++cursor;
+#line 46 "lib/Nana/toke.re"
+ { OP(TOKEN_LPAREN); }
+#line 92 "lib/Nana/toke.c"
+yy8:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '<': goto yy115;
+ case '=': goto yy113;
+ default: goto yy9;
+ }
+yy9:
+#line 77 "lib/Nana/toke.re"
+ { OP(TOKEN_GT); }
+#line 103 "lib/Nana/toke.c"
+yy10:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '=': goto yy111;
+ default: goto yy11;
+ }
+yy11:
+#line 49 "lib/Nana/toke.re"
+ { OP(TOKEN_DIV); }
+#line 113 "lib/Nana/toke.c"
+yy12:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '=': goto yy109;
+ default: goto yy13;
+ }
+yy13:
+#line 51 "lib/Nana/toke.re"
+ { OP(TOKEN_MOD); }
+#line 123 "lib/Nana/toke.c"
+yy14:
+ ++cursor;
+#line 52 "lib/Nana/toke.re"
+ { OP(TOKEN_COMMA); }
+#line 128 "lib/Nana/toke.c"
+yy16:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '=': goto yy107;
+ case '~': goto yy105;
+ default: goto yy17;
+ }
+yy17:
+#line 55 "lib/Nana/toke.re"
+ { OP(TOKEN_NOT); }
+#line 139 "lib/Nana/toke.c"
+yy18:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '=': goto yy103;
+ case '>': goto yy101;
+ case '~': goto yy99;
+ default: goto yy19;
+ }
+yy19:
+#line 59 "lib/Nana/toke.re"
+ { OP(TOKEN_ASSIGN); }
+#line 151 "lib/Nana/toke.c"
+yy20:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '=': goto yy97;
+ default: goto yy21;
+ }
+yy21:
+#line 61 "lib/Nana/toke.re"
+ { OP(TOKEN_XOR); }
+#line 161 "lib/Nana/toke.c"
+yy22:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '.': goto yy93;
+ default: goto yy23;
+ }
+yy23:
+#line 64 "lib/Nana/toke.re"
+ { OP(TOKEN_DOT); }
+#line 171 "lib/Nana/toke.c"
+yy24:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '=': goto yy87;
+ case '|': goto yy89;
+ default: goto yy25;
+ }
+yy25:
+#line 68 "lib/Nana/toke.re"
+ { OP(TOKEN_OR); }
+#line 182 "lib/Nana/toke.c"
+yy26:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '&': goto yy85;
+ case '=': goto yy83;
+ default: goto yy27;
+ }
+yy27:
+#line 71 "lib/Nana/toke.re"
+ { OP(TOKEN_AND); }
+#line 193 "lib/Nana/toke.c"
+yy28:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '=': goto yy77;
+ case '>': goto yy79;
+ default: goto yy29;
+ }
+yy29:
+#line 81 "lib/Nana/toke.re"
+ { OP(TOKEN_LT); }
+#line 204 "lib/Nana/toke.c"
+yy30:
+ ++cursor;
+#line 82 "lib/Nana/toke.re"
+ { OP(TOKEN_REF); }
+#line 209 "lib/Nana/toke.c"
+yy32:
+ ++cursor;
+#line 83 "lib/Nana/toke.re"
+ { OP(TOKEN_TILDE); }
+#line 214 "lib/Nana/toke.c"
+yy34:
+ yych = (char)*++cursor;
+ switch (yych) {
+ case '{': goto yy75;
+ default: goto yy5;
+ }
+yy35:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '*': goto yy71;
+ case '=': goto yy69;
+ default: goto yy36;
+ }
+yy36:
+#line 88 "lib/Nana/toke.re"
+ { OP(TOKEN_MUL); }
+#line 231 "lib/Nana/toke.c"
+yy37:
+ ++cursor;
+#line 92 "lib/Nana/toke.re"
+ { OP(TOKEN_LBRACE); }
+#line 236 "lib/Nana/toke.c"
+yy39:
+ yych = (char)*(marker = ++cursor);
+ switch (yych) {
+ case '!':
+ case '"':
+ case '\'':
+ case '(':
+ case '[':
+ case '{': goto yy57;
+ case 'q': goto yy62;
+ case 'r': goto yy61;
+ case 'w': goto yy59;
+ default: goto yy5;
+ }
+yy40:
+ ++cursor;
+#line 100 "lib/Nana/toke.re"
+ { OP(TOKEN_STRING_DQ); }
+#line 255 "lib/Nana/toke.c"
+yy42:
+ ++cursor;
+#line 101 "lib/Nana/toke.re"
+ { OP(TOKEN_STRING_SQ); }
+#line 260 "lib/Nana/toke.c"
+yy44:
+ ++cursor;
+#line 102 "lib/Nana/toke.re"
+ { OP(TOKEN_LBRACKET); }
+#line 265 "lib/Nana/toke.c"
+yy46:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '-': goto yy53;
+ case '=': goto yy49;
+ case '>': goto yy51;
+ case 'a':
+ case 'b':
+ case 'c':
+ case 'd':
+ case 'e':
+ case 'f':
+ case 'g':
+ case 'h':
+ case 'i':
+ case 'j':
+ case 'k':
+ case 'l':
+ case 'm':
+ case 'n':
+ case 'o':
+ case 'p':
+ case 'q':
+ case 'r':
+ case 's':
+ case 't':
+ case 'u':
+ case 'v':
+ case 'w':
+ case 'x':
+ case 'y':
+ case 'z': goto yy55;
+ default: goto yy47;
+ }
+yy47:
+#line 107 "lib/Nana/toke.re"
+ { OP(TOKEN_MINUS); }
+#line 303 "lib/Nana/toke.c"
+yy48:
+ yych = (char)*++cursor;
+ goto yy5;
+yy49:
+ ++cursor;
+#line 106 "lib/Nana/toke.re"
+ { OP(TOKEN_MINUS_ASSIGN); }
+#line 311 "lib/Nana/toke.c"
+yy51:
+ ++cursor;
+#line 105 "lib/Nana/toke.re"
+ { OP(TOKEN_LAMBDA); }
+#line 316 "lib/Nana/toke.c"
+yy53:
+ ++cursor;
+#line 104 "lib/Nana/toke.re"
+ { OP(TOKEN_MINUSMINUS); }
+#line 321 "lib/Nana/toke.c"
+yy55:
+ ++cursor;
+#line 103 "lib/Nana/toke.re"
+ { OP(TOKEN_FILETEST); }
+#line 326 "lib/Nana/toke.c"
+yy57:
+ ++cursor;
+#line 99 "lib/Nana/toke.re"
+ { OP(TOKEN_STRING_Q_START); }
+#line 331 "lib/Nana/toke.c"
+yy59:
+ yych = (char)*++cursor;
+ switch (yych) {
+ case '!':
+ case '"':
+ case '\'':
+ case '(':
+ case '[':
+ case '{': goto yy67;
+ default: goto yy60;
+ }
+yy60:
+ cursor = marker;
+ goto yy5;
+yy61:
+ yych = (char)*++cursor;
+ switch (yych) {
+ case '!':
+ case '"':
+ case '\'':
+ case '(':
+ case '[':
+ case '{': goto yy65;
+ default: goto yy60;
+ }
+yy62:
+ yych = (char)*++cursor;
+ switch (yych) {
+ case '!':
+ case '"':
+ case '\'':
+ case '(':
+ case '[':
+ case '{': goto yy63;
+ default: goto yy60;
+ }
+yy63:
+ ++cursor;
+#line 96 "lib/Nana/toke.re"
+ { OP(TOKEN_STRING_QQ_START); }
+#line 372 "lib/Nana/toke.c"
+yy65:
+ ++cursor;
+#line 97 "lib/Nana/toke.re"
+ { OP(TOKEN_REGEXP_QR_START); }
+#line 377 "lib/Nana/toke.c"
+yy67:
+ ++cursor;
+#line 98 "lib/Nana/toke.re"
+ { OP(TOKEN_QW_START); }
+#line 382 "lib/Nana/toke.c"
+yy69:
+ ++cursor;
+#line 87 "lib/Nana/toke.re"
+ { OP(TOKEN_MUL_ASSIGN); }
+#line 387 "lib/Nana/toke.c"
+yy71:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '=': goto yy73;
+ default: goto yy72;
+ }
+yy72:
+#line 86 "lib/Nana/toke.re"
+ { OP(TOKEN_POW); }
+#line 397 "lib/Nana/toke.c"
+yy73:
+ ++cursor;
+#line 85 "lib/Nana/toke.re"
+ { OP(TOKEN_POW_ASSIGN); }
+#line 402 "lib/Nana/toke.c"
+yy75:
+ ++cursor;
+#line 84 "lib/Nana/toke.re"
+ { OP(TOKEN_DEREF); }
+#line 407 "lib/Nana/toke.c"
+yy77:
+ ++cursor;
+#line 80 "lib/Nana/toke.re"
+ { OP(TOKEN_LE); }
+#line 412 "lib/Nana/toke.c"
+yy79:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '=': goto yy81;
+ default: goto yy80;
+ }
+yy80:
+#line 79 "lib/Nana/toke.re"
+ { OP(TOKEN_RSHIFT); }
+#line 422 "lib/Nana/toke.c"
+yy81:
+ ++cursor;
+#line 78 "lib/Nana/toke.re"
+ { OP(TOKEN_RSHIFT_ASSIGN); }
+#line 427 "lib/Nana/toke.c"
+yy83:
+ ++cursor;
+#line 70 "lib/Nana/toke.re"
+ { OP(TOKEN_AND_ASSIGN); }
+#line 432 "lib/Nana/toke.c"
+yy85:
+ ++cursor;
+#line 69 "lib/Nana/toke.re"
+ { OP(TOKEN_ANDAND); }
+#line 437 "lib/Nana/toke.c"
+yy87:
+ ++cursor;
+#line 67 "lib/Nana/toke.re"
+ { OP(TOKEN_OR_ASSIGN); }
+#line 442 "lib/Nana/toke.c"
+yy89:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '=': goto yy91;
+ default: goto yy90;
+ }
+yy90:
+#line 66 "lib/Nana/toke.re"
+ { OP(TOKEN_OROR); }
+#line 452 "lib/Nana/toke.c"
+yy91:
+ ++cursor;
+#line 65 "lib/Nana/toke.re"
+ { OP(TOKEN_OROR_ASSIGN); }
+#line 457 "lib/Nana/toke.c"
+yy93:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '.': goto yy95;
+ default: goto yy94;
+ }
+yy94:
+#line 63 "lib/Nana/toke.re"
+ { OP(TOKEN_DOTDOT); }
+#line 467 "lib/Nana/toke.c"
+yy95:
+ ++cursor;
+#line 62 "lib/Nana/toke.re"
+ { OP(TOKEN_DOTDOTDOT); }
+#line 472 "lib/Nana/toke.c"
+yy97:
+ ++cursor;
+#line 60 "lib/Nana/toke.re"
+ { OP(TOKEN_XOR_ASSIGN); }
+#line 477 "lib/Nana/toke.c"
+yy99:
+ ++cursor;
+#line 58 "lib/Nana/toke.re"
+ { OP(TOKEN_REGEXP_MATCH); }
+#line 482 "lib/Nana/toke.c"
+yy101:
+ ++cursor;
+#line 57 "lib/Nana/toke.re"
+ { OP(TOKEN_FAT_COMMA); }
+#line 487 "lib/Nana/toke.c"
+yy103:
+ ++cursor;
+#line 56 "lib/Nana/toke.re"
+ { OP(TOKEN_EQUAL_EQUAL); }
+#line 492 "lib/Nana/toke.c"
+yy105:
+ ++cursor;
+#line 54 "lib/Nana/toke.re"
+ { OP(TOKEN_REGEXP_NOT_MATCH); }
+#line 497 "lib/Nana/toke.c"
+yy107:
+ ++cursor;
+#line 53 "lib/Nana/toke.re"
+ { OP(TOKEN_NOT_EQUAL); }
+#line 502 "lib/Nana/toke.c"
+yy109:
+ ++cursor;
+#line 50 "lib/Nana/toke.re"
+ { OP(TOKEN_MOD_ASSIGN); }
+#line 507 "lib/Nana/toke.c"
+yy111:
+ ++cursor;
+#line 48 "lib/Nana/toke.re"
+ { OP(TOKEN_DIV_ASSIGN); }
+#line 512 "lib/Nana/toke.c"
+yy113:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '>': goto yy121;
+ default: goto yy114;
+ }
+yy114:
+#line 76 "lib/Nana/toke.re"
+ { OP(TOKEN_GE); }
+#line 522 "lib/Nana/toke.c"
+yy115:
+ ++cursor;
+ switch ((yych = (char)*cursor)) {
+ case '\'': goto yy117;
+ case '=': goto yy119;
+ default: goto yy116;
+ }
+yy116:
+#line 74 "lib/Nana/toke.re"
+ { OP(TOKEN_LSHIFT); }
+#line 533 "lib/Nana/toke.c"
+yy117:
+ ++cursor;
+#line 47 "lib/Nana/toke.re"
+ { OP(TOKEN_HEREDOC_SQ_START); }
+#line 538 "lib/Nana/toke.c"
+yy119:
+ ++cursor;
+#line 72 "lib/Nana/toke.re"
+ { OP(TOKEN_LSHIFT_ASSIGN); }
+#line 543 "lib/Nana/toke.c"
+yy121:
+ ++cursor;
+#line 75 "lib/Nana/toke.re"
+ { OP(TOKEN_CMP); }
+#line 548 "lib/Nana/toke.c"
+yy123:
+ ++cursor;
+#line 44 "lib/Nana/toke.re"
+ { OP(TOKEN_BYTES_DQ); }
+#line 553 "lib/Nana/toke.c"
+yy125:
+ ++cursor;
+#line 45 "lib/Nana/toke.re"
+ { OP(TOKEN_BYTES_SQ); }
+#line 558 "lib/Nana/toke.c"
+yy127:
+ ++cursor;
+#line 42 "lib/Nana/toke.re"
+ { OP(TOKEN_PLUS_ASSIGN); }
+#line 563 "lib/Nana/toke.c"
+yy129:
+ ++cursor;
+#line 41 "lib/Nana/toke.re"
+ { OP(TOKEN_PLUSPLUS); }
+#line 568 "lib/Nana/toke.c"
+ }
+#line 110 "lib/Nana/toke.re"
+
+ abort(); /* should not reach here */
+}
View
112 lib/Nana/toke.re
@@ -0,0 +1,112 @@
+/* vim: set filetype=cpp: */
+#include<stdio.h>
+
+/**
+ * Take a token from string.
+ *
+ * @args src: source string.
+ * @args len: length for 'src'.
+ * @args output *used: used chars in src
+ * @args output *found_end: make true if found __END__
+ * @args output *lineno_inc: incremented line numbers.
+ * @return int token id.
+ */
+int token_op(char *src, size_t len, int *used, int *found_end, int *lineno_inc) {
+#define OP(type) do { *used+=(cursor-orig); return type; } while (0)
+ *used = skip_ws(src, len, found_end, lineno_inc);
+ if (*found_end) {
+ return TOKEN_EOF;
+ }
+ if (*used == len) {
+ return TOKEN_EOF;
+ }
+ char *cursor = src+*used;
+ char *orig = cursor;
+ char *marker = cursor;
+
+ /*!re2c
+ re2c:define:YYCTYPE = "char";
+ re2c:define:YYCURSOR = cursor;
+ re2c:define:YYMARKER = marker;
+ re2c:yyfill:enable = 0;
+ re2c:yych:conversion = 1;
+ re2c:indent:top = 1;
+
+ OPENCHAR = [!'\{\["\(];
+ ANY_CHAR = [^];
+
+ */
+
+ /*!re2c
+ "++" { OP(TOKEN_PLUSPLUS); }
+ "+=" { OP(TOKEN_PLUS_ASSIGN); }
+ "+" { OP(TOKEN_PLUS); }
+ "b\"" { OP(TOKEN_BYTES_DQ); }
+ "b\'" { OP(TOKEN_BYTES_SQ); }
+ "(" { OP(TOKEN_LPAREN); }
+ "<<'" { OP(TOKEN_HEREDOC_SQ_START); }
+ "/=" { OP(TOKEN_DIV_ASSIGN); }
+ "/" { OP(TOKEN_DIV); }
+ "%=" { OP(TOKEN_MOD_ASSIGN); }
+ "%" { OP(TOKEN_MOD); }
+ "," { OP(TOKEN_COMMA); }
+ "!=" { OP(TOKEN_NOT_EQUAL); }
+ "!~" { OP(TOKEN_REGEXP_NOT_MATCH); }
+ "!" { OP(TOKEN_NOT); }
+ "==" { OP(TOKEN_EQUAL_EQUAL); }
+ "=>" { OP(TOKEN_FAT_COMMA); }
+ "=~" { OP(TOKEN_REGEXP_MATCH); }
+ "=" { OP(TOKEN_ASSIGN); }
+ "^=" { OP(TOKEN_XOR_ASSIGN); }
+ "^" { OP(TOKEN_XOR); }
+ "..." { OP(TOKEN_DOTDOTDOT); }
+ ".." { OP(TOKEN_DOTDOT); }
+ "." { OP(TOKEN_DOT); }
+ "||=" { OP(TOKEN_OROR_ASSIGN); }
+ "||" { OP(TOKEN_OROR); }
+ "|=" { OP(TOKEN_OR_ASSIGN); }
+ "|" { OP(TOKEN_OR); }
+ "&&" { OP(TOKEN_ANDAND); }
+ "&=" { OP(TOKEN_AND_ASSIGN); }
+ "&" { OP(TOKEN_AND); }
+ "<<=" { OP(TOKEN_LSHIFT_ASSIGN); }
+ "<<'" { OP(TOKEN_HEREDOC_SQ_START); }
+ "<<" { OP(TOKEN_LSHIFT); }
+ "<=>" { OP(TOKEN_CMP); }
+ "<=" { OP(TOKEN_GE); }
+ "<" { OP(TOKEN_GT); }
+ ">>=" { OP(TOKEN_RSHIFT_ASSIGN); }
+ ">>" { OP(TOKEN_RSHIFT); }
+ ">=" { OP(TOKEN_LE); }
+ ">" { OP(TOKEN_LT); }
+ "\\" { OP(TOKEN_REF); }
+ "~" { OP(TOKEN_TILDE); }
+ "${" { OP(TOKEN_DEREF); }
+ "**=" { OP(TOKEN_POW_ASSIGN); }
+ "**" { OP(TOKEN_POW); }
+ "*=" { OP(TOKEN_MUL_ASSIGN); }
+ "*" { OP(TOKEN_MUL); }
+ "++" { OP(TOKEN_PLUSPLUS); }
+ "+=" { OP(TOKEN_PLUS_ASSIGN); }
+ "+" { OP(TOKEN_PLUS); }
+ "{" { OP(TOKEN_LBRACE); }
+ "(" { OP(TOKEN_LPAREN); }
+ "b\'" { OP(TOKEN_BYTES_SQ); }
+ "b\"" { OP(TOKEN_BYTES_DQ); }
+ "qq" OPENCHAR { OP(TOKEN_STRING_QQ_START); }
+ "qr" OPENCHAR { OP(TOKEN_REGEXP_QR_START); }
+ "qw" OPENCHAR { OP(TOKEN_QW_START); }
+ "q" OPENCHAR { OP(TOKEN_STRING_Q_START); }
+ "\"" { OP(TOKEN_STRING_DQ); }
+ "'" { OP(TOKEN_STRING_SQ); }
+ "[" { OP(TOKEN_LBRACKET); }
+ "-" [a-z] { OP(TOKEN_FILETEST); }
+ "--" { OP(TOKEN_MINUSMINUS); }
+ "->" { OP(TOKEN_LAMBDA); }
+ "-=" { OP(TOKEN_MINUS_ASSIGN); }
+ "-" { OP(TOKEN_MINUS); }
+ ANY_CHAR { return TOKEN_EOF; }
+
+ */
+ abort(); /* should not reach here */
+}
View
1  lib/Nana/toke.re.PL
@@ -0,0 +1 @@
+system("re2c -o lib/Nana/toke.c lib/Nana/toke.re");
View
15 t/07_tokenize.t
@@ -362,3 +362,18 @@ TOKEN_LE
--- src: qq{
--- token: TOKEN_STRING_QQ_START
+===
+--- src: <<'
+--- token: TOKEN_HEREDOC_SQ_START
+
+===
+--- src: b'
+--- token: TOKEN_BYTES_SQ
+
+===
+--- src: b"
+--- token: TOKEN_BYTES_DQ
+
+===
+--- src: (
+--- token: TOKEN_LPAREN
View
7 util/myprove
@@ -0,0 +1,7 @@
+use strict;
+use App::Prove;
+use FindBin;
+use lib "$FindBin::Bin/../util/";
+
+my $app = App::Prove->new;$app->process_args(@ARGV);
+exit( $app->run ? 0 : 1 );

No commit comments for this range

Something went wrong with that request. Please try again.