From cd73b1c967ab33740c8f50ff685e6fcb7d911fdb Mon Sep 17 00:00:00 2001
From: "Ahmad M. Zawawi" <ahmad.zawawi@gmail.com>
Date: Mon, 19 Nov 2012 13:35:34 +0300
Subject: [PATCH] Added Lexer example script (for future research)

---
 test_lex.pl | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100755 test_lex.pl

diff --git a/test_lex.pl b/test_lex.pl
new file mode 100755
index 0000000..f28c84c
--- /dev/null
+++ b/test_lex.pl
@@ -0,0 +1,27 @@
+#!/usr/bin/env perl
+
+use Modern::Perl;
+
+my $target = q{say "Hello world"};
+
+use Data::Printer;
+
+my @tokens;
+while(1) {
+	if($target =~ /\G(([\$\@\%])?[A-Za-z_]([A-Za-z_]|(\-[A-Za-z_]))+)/gc) {
+		push @tokens, ['IDENTIFIER', $1, pos($target) - length($1)];
+	} elsif($target =~ /\G(\".+\")/gc) {
+		push @tokens, ['DOUBLE_QUOTE', $1, pos($target) - length($1)];
+	} elsif($target =~ /\G([\-\+\*\/])/gc) { 
+		push @tokens, ['OPERATOR', $1, pos($target) - length($1)];
+	} elsif($target =~ /\G(\d+)/gc) {
+		push @tokens, ['INTEGER', $1, pos($target) - length($1)];
+	} elsif( $target =~ /\G(\s+)/gc) {
+		push @tokens, ['WHITESPACE', $1, pos($target) - length($1)];
+	} else {
+		last;
+	}
+}
+
+p($target);
+p(@tokens)