Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Added Lexer example script (for future research)

  • Loading branch information...
commit cd73b1c967ab33740c8f50ff685e6fcb7d911fdb 1 parent 78a0507
@azawawi authored
Showing with 27 additions and 0 deletions.
  1. +27 −0 test_lex.pl
View
27 test_lex.pl
@@ -0,0 +1,27 @@
+#!/usr/bin/env perl
+
+use Modern::Perl;
+
+my $target = q{say "Hello world"};
+
+use Data::Printer;
+
+my @tokens;
+while(1) {
+ if($target =~ /\G(([\$\@\%])?[A-Za-z_]([A-Za-z_]|(\-[A-Za-z_]))+)/gc) {
+ push @tokens, ['IDENTIFIER', $1, pos($target) - length($1)];
+ } elsif($target =~ /\G(\".+\")/gc) {
+ push @tokens, ['DOUBLE_QUOTE', $1, pos($target) - length($1)];
+ } elsif($target =~ /\G([\-\+\*\/])/gc) {
+ push @tokens, ['OPERATOR', $1, pos($target) - length($1)];
+ } elsif($target =~ /\G(\d+)/gc) {
+ push @tokens, ['INTEGER', $1, pos($target) - length($1)];
+ } elsif( $target =~ /\G(\s+)/gc) {
+ push @tokens, ['WHITESPACE', $1, pos($target) - length($1)];
+ } else {
+ last;
+ }
+}
+
+p($target);
+p(@tokens)
Please sign in to comment.
Something went wrong with that request. Please try again.