Skip to content
Browse files

Moved files / Added example

  • Loading branch information...
1 parent 97c62b8 commit e599553ffe177005d08d85062cbd456cf0d67c1a @pstuifzand committed Mar 16, 2012
View
5 examples/add/add.mp
@@ -0,0 +1,5 @@
+
+Number = /\d+/
+
+
+Parser ::= Number {{ return $_[1]; }}
View
0 Tokenizer.pm → old/Tokenizer.pm
File renamed without changes.
View
0 Tokenizer/Actions.pm → old/Tokenizer/Actions.pm
File renamed without changes.
View
199 old/output.pl
@@ -0,0 +1,199 @@
+use v5.10;
+use strict;
+use Marpa::XS;
+use Data::Dumper;
+use My_Actions;
+
+$Data::Dumper::Deepcopy = 1;
+
+my %tokens = (
+ Name => qr/(\w+)/,
+ DeclareOp => qr/::=/,
+ Plus => qr/\+/,
+ Star => qr/\*/,
+ CB => qr/{{/,
+ CE => qr/}}/,
+ Code => qr/(.+)(?=}})/,
+);
+
+sub My_Actions::Lhs_0 {
+ shift; return [ lhs => $_[0] ]
+}
+sub My_Actions::Names_0 {
+ shift; return [ @_ ];
+}
+sub My_Actions::Rhs_0 {
+ shift; return [ rhs => $_[0] ]
+}
+sub My_Actions::Rhs_1 {
+ shift; return [ rhs => $_[0], min => 1 ]
+}
+sub My_Actions::Rhs_2 {
+ shift; return [ rhs => $_[0], min => 0 ]
+}
+sub My_Actions::Parser_0 {
+ shift; return { 'rules' => \@_ }
+}
+sub My_Actions::Rule_0 {
+ shift; return { @{$_[0]}, @{$_[2]} }
+}
+sub create_grammar {
+ my $grammar = Marpa::XS::Grammar->new(
+ { start => 'Parser',
+ actions => 'My_Actions',
+
+ 'rules' => [
+ {
+ 'min' => 1,
+ 'rhs' => [
+ 'Rule'
+ ],
+ 'lhs' => 'Parser',
+ 'action' => 'Parser_0'
+ },
+ {
+ 'rhs' => [
+ 'Lhs',
+ 'DeclareOp',
+ 'Rhs'
+ ],
+ 'lhs' => 'Rule',
+ 'action' => 'Rule_0'
+ },
+ {
+ 'rhs' => [
+ 'Name'
+ ],
+ 'lhs' => 'Lhs',
+ 'action' => 'Lhs_0'
+ },
+ {
+ 'rhs' => [
+ 'Names'
+ ],
+ 'lhs' => 'Rhs',
+ 'action' => 'Rhs_0'
+ },
+ {
+ 'rhs' => [
+ 'Names',
+ 'Plus'
+ ],
+ 'lhs' => 'Rhs',
+ 'action' => 'Rhs_1'
+ },
+ {
+ 'rhs' => [
+ 'Names',
+ 'Star'
+ ],
+ 'lhs' => 'Rhs',
+ 'action' => 'Rhs_2'
+ },
+ {
+ 'min' => 1,
+ 'rhs' => [
+ 'Name'
+ ],
+ 'lhs' => 'Names',
+ 'action' => 'Names_0'
+ }
+ ],
+ terminals => [keys %tokens],
+ },
+ );
+ $grammar->precompute();
+ return $grammar;
+}
+
+sub parse_token_stream {
+ my ($grammar, $fh) = @_;
+
+ my $r= Marpa::XS::Recognizer->new( { grammar => $grammar } );
+
+ LINE: while (<$fh>) {
+ my $line = $_;
+ chomp $line;
+ print $line;
+
+ while ($line) {
+ $line =~ s/^\s+//;
+ next LINE if $line =~ m/^\#/;
+ #say $line;
+
+ for my $token_name (@{$r->terminals_expected}) {
+ #say $token_name;
+ my $re = $tokens{$token_name};
+
+ if ($line =~ s/^$re//) {
+ $r->read($token_name, $1 ? $1 : '');
+ }
+ }
+ }
+ }
+
+ my $value_ref = $r->value;
+ return $$value_ref;
+}
+
+sub generate_parser_code {
+ my ($parse_tree, $config) = @_;
+
+ my $namespace = $config->{namespace};
+
+ my $out = <<"PRE";
+sub create_grammar {
+ my \$grammar = Marpa::XS::Grammar->new(
+ { start => 'Parser',
+ actions => '$namespace',
+PRE
+ $out .= generate_rules($parse_tree, $config);
+
+ $out .= <<'POST';
+ );
+ $grammar->precompute();
+ return $grammar;
+}
+POST
+}
+
+sub generate_rules {
+ my ($parse_tree) = @_;
+ my $out = Dumper($parse_tree);
+ $out =~ s/\$VAR\d+\s+=\s+{//;
+ $out =~ s/};\n$/}/s;
+ return $out;
+}
+
+sub generate_actions {
+ my ($parse_tree, $config) = @_;
+ my %actions;
+
+ for (@{ $parse_tree->{rules} }) {
+ my $c = @{ $actions{$_->{lhs}} || [] };
+ my $name = $_->{lhs}.'_'.$c;
+ $_->{action} = $name;
+ push @{ $actions{$_->{lhs}} }, { name => $name, code => $_->{code} };
+ delete $_->{code};
+ }
+
+ my $namespace = $config->{namespace};
+
+ for my $rule_name (keys %actions) {
+ for my $action (@{$actions{$rule_name}}) {
+ say "sub ${namespace}::$action->{name} {";
+ say "\t".$action->{code};
+ say "}";
+ }
+ }
+}
+
+open my $fh, '<', $ARGV[0] or die "Can't open $ARGV[0]";
+
+my $grammar = create_grammar();
+my $parse_tree = parse_token_stream($grammar, $fh);
+
+my $config = { namespace => 'My_Actions' };
+generate_actions($parse_tree, $config);
+print generate_parser_code($parse_tree, $config);
+
View
210 old/output2.pl
@@ -0,0 +1,210 @@
+use v5.10;
+use strict;
+use Marpa::XS;
+use Data::Dumper;
+use My_Actions;
+
+$Data::Dumper::Deepcopy = 1;
+
+my %tokens = (
+ Name => qr/(\w+)/,
+ DeclareOp => qr/::=/,
+ Plus => qr/\+/,
+ Star => qr/\*/,
+ CB => qr/{{/,
+ CE => qr/}}/,
+ Code => qr/(.+)(?=}})/,
+);
+
+sub My_Actions::Lhs_0 {
+ shift; return [ lhs => $_[0] ]
+}
+sub My_Actions::Names_0 {
+ shift; return [ @_ ];
+}
+sub My_Actions::Rhs_0 {
+ shift; return [ rhs => $_[0] ]
+}
+sub My_Actions::Rhs_1 {
+ shift; return [ rhs => $_[0], min => 1 ]
+}
+sub My_Actions::Rhs_2 {
+ shift; return [ rhs => $_[0], min => 0 ]
+}
+sub My_Actions::Parser_0 {
+ shift; return { 'rules' => \@_ }
+}
+sub My_Actions::Rule_0 {
+ shift; return { @{$_[0]}, @{$_[2]} }
+}
+sub My_Actions::Rule_1 {
+ shift; return { @{$_[0]}, @{$_[2]}, code => $_[4] }
+}
+sub create_grammar {
+ my $grammar = Marpa::XS::Grammar->new(
+ { start => 'Parser',
+ actions => 'My_Actions',
+
+ 'rules' => [
+ {
+ 'min' => 1,
+ 'rhs' => [
+ 'Rule'
+ ],
+ 'lhs' => 'Parser',
+ 'action' => 'Parser_0'
+ },
+ {
+ 'rhs' => [
+ 'Lhs',
+ 'DeclareOp',
+ 'Rhs'
+ ],
+ 'lhs' => 'Rule',
+ 'action' => 'Rule_0'
+ },
+ {
+ 'rhs' => [
+ 'Lhs',
+ 'DeclareOp',
+ 'Rhs',
+ 'CB',
+ 'Code',
+ 'CE'
+ ],
+ 'lhs' => 'Rule',
+ 'action' => 'Rule_1'
+ },
+ {
+ 'rhs' => [
+ 'Name'
+ ],
+ 'lhs' => 'Lhs',
+ 'action' => 'Lhs_0'
+ },
+ {
+ 'rhs' => [
+ 'Names'
+ ],
+ 'lhs' => 'Rhs',
+ 'action' => 'Rhs_0'
+ },
+ {
+ 'rhs' => [
+ 'Names',
+ 'Plus'
+ ],
+ 'lhs' => 'Rhs',
+ 'action' => 'Rhs_1'
+ },
+ {
+ 'rhs' => [
+ 'Names',
+ 'Star'
+ ],
+ 'lhs' => 'Rhs',
+ 'action' => 'Rhs_2'
+ },
+ {
+ 'min' => 1,
+ 'rhs' => [
+ 'Name'
+ ],
+ 'lhs' => 'Names',
+ 'action' => 'Names_0'
+ }
+ ],
+ terminals => [keys %tokens],
+ } );
+ $grammar->precompute();
+ return $grammar;
+}
+
+sub parse_token_stream {
+ my ($grammar, $fh) = @_;
+
+ my $r= Marpa::XS::Recognizer->new( { grammar => $grammar } );
+
+ LINE: while (<$fh>) {
+ my $line = $_;
+ chomp $line;
+
+ while ($line) {
+ $line =~ s/^\s+//;
+ next LINE if $line =~ m/^\#/;
+
+ for my $token_name (@{$r->terminals_expected}) {
+ my $re = $tokens{$token_name};
+
+ if ($line =~ s/^$re//) {
+ $r->read($token_name, $1 ? $1 : '');
+ }
+ }
+ }
+ }
+
+ my $value_ref = $r->value;
+ return $$value_ref;
+}
+
+sub generate_parser_code {
+ my ($parse_tree, $config) = @_;
+
+ my $namespace = $config->{namespace};
+
+ my $out = <<"PRE";
+sub create_grammar {
+ my \$grammar = Marpa::XS::Grammar->new(
+ { start => 'Parser',
+ actions => '$namespace',
+PRE
+ $out .= generate_rules($parse_tree, $config);
+
+ $out .= <<'POST';
+ );
+ $grammar->precompute();
+ return $grammar;
+}
+POST
+}
+
+sub generate_rules {
+ my ($parse_tree) = @_;
+ my $out = Dumper($parse_tree);
+ $out =~ s/\$VAR\d+\s+=\s+{//;
+ $out =~ s/};\n$/}/s;
+ return $out;
+}
+
+sub generate_actions {
+ my ($parse_tree, $config) = @_;
+ my %actions;
+
+ for (@{ $parse_tree->{rules} }) {
+ my $c = @{ $actions{$_->{lhs}} || [] };
+ my $name = $_->{lhs}.'_'.$c;
+ $_->{action} = $name;
+ push @{ $actions{$_->{lhs}} }, { name => $name, code => $_->{code} };
+ delete $_->{code};
+ }
+
+ my $namespace = $config->{namespace};
+
+ for my $rule_name (keys %actions) {
+ for my $action (@{$actions{$rule_name}}) {
+ say "sub ${namespace}::$action->{name} {";
+ say "\t".$action->{code};
+ say "}";
+ }
+ }
+}
+
+open my $fh, '<', $ARGV[0] or die "Can't open $ARGV[0]";
+
+my $grammar = create_grammar();
+my $parse_tree = parse_token_stream($grammar, $fh);
+
+my $config = { namespace => 'My_Actions' };
+generate_actions($parse_tree, $config);
+print generate_parser_code($parse_tree, $config);
+
View
0 tokenizer.pl → old/tokenizer.pl
File renamed without changes.
View
50 old/tokens.pl
@@ -0,0 +1,50 @@
+use v5.10;
+use Marpa::XS;
+use Data::Dumper;
+
+my $grammar = Marpa::XS::Grammar->new(
+ { start => 'Parser',
+ actions => 'My_Actions',
+ rules => [
+ { lhs => 'Parser', rhs => [ 'NumberLine' ], min => 1, action => 'Parser' },
+ { lhs => 'NumberLine', rhs => [ 'NumberList', "\n" ], action => 'NumberLine' },
+ { lhs => 'NumberList', rhs => ['Number'] },
+ { lhs => 'NumberList', rhs => ['Number', 'WS', 'NumberList'] },
+ { lhs => 'Number', rhs => ['Digit'], min => 1, action => 'Number' },
+ { lhs => 'Digit', rhs => ['0'] },
+ { lhs => 'Digit', rhs => ['1'] },
+ { lhs => 'Digit', rhs => ['2'] },
+ { lhs => 'Digit', rhs => ['3'] },
+ { lhs => 'Digit', rhs => ['4'] },
+ { lhs => 'Digit', rhs => ['5'] },
+ { lhs => 'Digit', rhs => ['6'] },
+ { lhs => 'Digit', rhs => ['7'] },
+ { lhs => 'Digit', rhs => ['8'] },
+ { lhs => 'Digit', rhs => ['9'] },
+ { lhs => 'WS', rhs => [ ' ' ], min => 1 },
+ ],
+ }
+);
+use List::Util 'sum';
+
+sub My_Actions::Parser { shift;return [ map { if (ref) { @$_ } else { $_ } } @_ ]; }
+sub My_Actions::NumberLine { shift; pop; return [ map { if (ref) { @$_ } else { $_ } } @_ ]; }
+sub My_Actions::NumberList { shift; return [ $_[0] ] if (@_ == 1); return [ $_[0], @{$_[2]} ] if @_ == 3; }
+sub My_Actions::Number { shift; return [ 'Number', (join '', @_) ] }
+sub My_Actions::Digit { shift; return $_[0]; }
+sub My_Actions::WS { shift; return; }
+
+$grammar->precompute();
+
+my $re = Marpa::XS::Recognizer->new( { grammar => $grammar } );
+
+while (<>) {
+ my @tokens = split //;
+ for (@tokens) {
+ $re->read($_, $_);
+ }
+}
+
+my $value_ref = $re->value;
+say Dumper($$value_ref);
+
View
3 test.mt
@@ -1,3 +0,0 @@
-Regex=R:R:(.*)$
-Name=R:(\w+)
-DeclareOp=R:=

0 comments on commit e599553

Please sign in to comment.
Something went wrong with that request. Please try again.