Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Cleanup tokenizer
- use names
- use data driven terminals
- if token is not done then restart there
  • Loading branch information
pstuifzand committed Jun 21, 2012
1 parent 821d437 commit 2ce6bf6
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 38 deletions.
61 changes: 28 additions & 33 deletions lib/MarpaX/Simple/Rules.pm
Expand Up @@ -29,14 +29,14 @@ sub parse_rules {
rules => [
{ lhs => 'Rules', rhs => [qw/Rule/], action => 'Rules', min => 1 },
{ lhs => 'Rule', rhs => [qw/Lhs/], action => 'MissingRHS' },
{ lhs => 'Rule', rhs => [qw/::=/], action => 'MissingLHS' },
{ lhs => 'Rule', rhs => [qw/Lhs ::= Rhs/], action => 'Rule' },
{ lhs => 'Rule', rhs => [qw/Lhs ::= Rhs => ActionName/], action => 'RuleWithAction' },
{ lhs => 'Rule', rhs => [qw/Lhs ::= Rhs => Name/], action => 'RuleWithAction' },
{ lhs => 'Rule', rhs => [qw/DeclareOp/], action => 'MissingLHS' },
{ lhs => 'Rule', rhs => [qw/Lhs DeclareOp Rhs/], action => 'Rule' },
{ lhs => 'Rule', rhs => [qw/Lhs DeclareOp Rhs ActionArrow ActionName/], action => 'RuleWithAction' },
{ lhs => 'Rule', rhs => [qw/Lhs DeclareOp Rhs ActionArrow Name/], action => 'RuleWithAction' },
{ lhs => 'Lhs', rhs => [qw/Name/], action => 'Lhs' },
{ lhs => 'Rhs', rhs => [qw/Names/], action => 'Rhs' },
{ lhs => 'Rhs', rhs => [qw/Name +/], action => 'Plus' },
{ lhs => 'Rhs', rhs => [qw/Name */], action => 'Star' },
{ lhs => 'Rhs', rhs => [qw/Name Plus/], action => 'Plus' },
{ lhs => 'Rhs', rhs => [qw/Name Star/], action => 'Star' },
{ lhs => 'Rhs', rhs => [qw/Null/], action => 'Null' },
{ lhs => 'Names', rhs => [qw/Name/], action => 'Names', min => 1 },
],
Expand All @@ -52,33 +52,28 @@ sub parse_rules {
return [];
}

for (@tokens) {
next if m/^\s*$/;

if (m/^::=$/) {
$rec->read('::=');
}
if (m/^(::(whatever|undef))$/) {
$rec->read('ActionName', $1);
}
elsif (m/^Null$/) {
$rec->read('Null');
}
elsif (m/^=>$/) {
$rec->read('=>');
}
elsif (m/^[+]$/) {
$rec->read('+');
}
elsif (m/^[*]$/) {
$rec->read('*');
}
elsif (m/^(\w+)$/) {
$rec->read('Name', $1);
}
elsif (m/^(\w+)([+*]?)$/) {
$rec->read('Name', $1);
$rec->read($2, $2);
my @terminals = (
[ 'DeclareOp', '::=' ],
[ 'ActionName', qr/(::(whatever|undef))/ ],
[ 'Null', 'Null' ],
[ 'ActionArrow', '=>' ],
[ 'Plus', '\+' ],
[ 'Star', '\*' ],
[ 'Name', qr/\w+/, ],
);

TOKEN: for my $token (@tokens) {
next if $token =~ m/^\s*$/;

for my $t (@terminals) {
if ($token =~ m/^($t->[1])/) {
$rec->read($t->[0], $2 // $1);
$token =~ s/$t->[1]//;
if ($token) {
redo TOKEN;
}
next TOKEN;
}
}
}

Expand Down
7 changes: 2 additions & 5 deletions t/whitespace.t
Expand Up @@ -58,13 +58,10 @@ RULES

# No whitespace around operator
{
eval {
parse_rules(<<"RULES");
my $rules = parse_rules(<<"RULES");
a::=b
RULES
};
unlike($@, qr/Can't use an undefined value as a SCALAR reference/);
like($@, qr/Can't parse/);
is_deeply($rules, [ { lhs => 'a', rhs => [qw/b/] } ]);
}
done_testing();

0 comments on commit 2ce6bf6

Please sign in to comment.